{
  "schemaVersion": 1,
  "deviceProperties": [
    {
      "id": 0, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 1, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 2, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 3, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 4, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 5, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 6, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 7, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    }
  ],
  "cupti_version": 22,
  "cuda_runtime_version": 12040,
  "cuda_driver_version": 12080,
  "distributedInfo": {"backend": "nccl", "rank": 3, "world_size": 8, "pg_count": 1, "pg_config": [{"pg_name": "0", "pg_desc": "default_pg", "backend_config": "cuda:nccl", "pg_size": 8, "ranks": [0, 1, 2, 3, 4, 5, 6, 7]}], "nccl_version": "2.21.5"},
  "record_shapes": 1,
  "trace_id": "FDAB8076CAC140AE88AB0C065CAFD7BD",
  "traceEvents": [
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: DivBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704354.621, "dur": 123.259,
    "args": {
      "External id": 2466305,"Record function id": 0, "Sequence number": 24740650, "Fwd thread id": 1, "Ev Idx": 0
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "DivBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704376.491, "dur": 91.840,
    "args": {
      "External id": 2466306,"Sequence number": 24740650, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 1
    }
  },
  {
    "ph": "f", "id": 1, "pid": 1336756, "tid": 1381189, "ts": 1514656704376.491,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704384.630, "dur": 81.600,
    "args": {
      "External id": 2466307,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 2
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704490.779, "dur": 257.478,
    "args": {
      "External id": 2466308,"Record function id": 0, "Ev Idx": 3
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704564.114, "dur": 103.592,
    "args": {
      "External id": 2466309,"Record function id": 0, "Ev Idx": 4
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.27", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704601.754, "dur": 54.731,
    "args": {
      "External id": 2466310,"Record function id": 0, "Ev Idx": 5
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704673.368, "dur": 1.702,
    "args": {
      "External id": 2466311,"Sequence number": 24740649, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 6
    }
  },
  {
    "ph": "f", "id": 2, "pid": 1336756, "tid": 1381189, "ts": 1514656704673.368,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704679.968, "dur": 62.151,
    "args": {
      "External id": 2466312,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 7
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704690.828, "dur": 50.773,
    "args": {
      "External id": 2466313,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 8
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704702.508, "dur": 3.771,
    "args": {
      "External id": 2466314,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704759.315, "dur": 36586.745,
    "args": {
      "External id": 2466315,"Record function id": 0, "Sequence number": 24740647, "Fwd thread id": 1, "Ev Idx": 10
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704760.898, "dur": 36573.517,
    "args": {
      "External id": 2466316,"Sequence number": 24740647, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11
    }
  },
  {
    "ph": "f", "id": 3, "pid": 1336756, "tid": 1381189, "ts": 1514656704760.898,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704805.813, "dur": 3.754,
    "args": {
      "External id": 2466317,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 12
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704814.973, "dur": 36399.392,
    "args": {
      "External id": 2466318,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 13
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704816.929, "dur": 36397.081,
    "args": {
      "External id": 2466319,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 14
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704822.243, "dur": 6.846,
    "args": {
      "External id": 2466320,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 15
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656704831.262, "dur": 36380.856,
    "args": {
      "External id": 2466321,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 16
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741219.545, "dur": 0.413,
    "args": {
      "External id": 2466322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 17
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741222.798, "dur": 3.334,
    "args": {
      "External id": 2466323,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 18
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741224.514, "dur": 1.127,
    "args": {
      "External id": 2466324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 19
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741233.056, "dur": 34.405,
    "args": {
      "External id": 2466325,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 20
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741276.454, "dur": 47.582,
    "args": {
      "External id": 2466326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 21
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741278.357, "dur": 45.477,
    "args": {
      "External id": 2466327,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 22
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741280.735, "dur": 42.711,
    "args": {
      "External id": 2466328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 23
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741358.006, "dur": 20.206,
    "args": {
      "External id": 2466329,"Record function id": 0, "Sequence number": 24740646, "Fwd thread id": 1, "Ev Idx": 24
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741359.880, "dur": 15.402,
    "args": {
      "External id": 2466330,"Sequence number": 24740646, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 25
    }
  },
  {
    "ph": "f", "id": 4, "pid": 1336756, "tid": 1381189, "ts": 1514656741359.880,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741364.194, "dur": 10.870,
    "args": {
      "External id": 2466331,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 26
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741368.211, "dur": 6.647,
    "args": {
      "External id": 2466332,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 27
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741382.244, "dur": 113.362,
    "args": {
      "External id": 2466333,"Record function id": 0, "Sequence number": 24740645, "Fwd thread id": 1, "Ev Idx": 28
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741383.372, "dur": 102.834,
    "args": {
      "External id": 2466334,"Sequence number": 24740645, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 29
    }
  },
  {
    "ph": "f", "id": 5, "pid": 1336756, "tid": 1381189, "ts": 1514656741383.372,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741387.144, "dur": 98.557,
    "args": {
      "External id": 2466335,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 30
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741393.445, "dur": 37.889,
    "args": {
      "External id": 2466336,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 31
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741398.705, "dur": 5.810,
    "args": {
      "External id": 2466337,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 32
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741406.799, "dur": 24.214,
    "args": {
      "External id": 2466338,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 33
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741410.331, "dur": 20.166,
    "args": {
      "External id": 2466339,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 34
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741436.535, "dur": 4.445,
    "args": {
      "External id": 2466340,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 35
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741439.012, "dur": 1.536,
    "args": {
      "External id": 2466341,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 36
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741442.626, "dur": 42.184,
    "args": {
      "External id": 2466342,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 37
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741500.294, "dur": 85.542,
    "args": {
      "External id": 2466343,"Record function id": 0, "Sequence number": 24740644, "Fwd thread id": 1, "Ev Idx": 38
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741501.717, "dur": 79.984,
    "args": {
      "External id": 2466344,"Sequence number": 24740644, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 39
    }
  },
  {
    "ph": "f", "id": 6, "pid": 1336756, "tid": 1381189, "ts": 1514656741501.717,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741504.825, "dur": 76.584,
    "args": {
      "External id": 2466345,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "3"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 40
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741510.184, "dur": 35.567,
    "args": {
      "External id": 2466346,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 41
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741511.105, "dur": 3.282,
    "args": {
      "External id": 2466347,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 42
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741515.117, "dur": 30.374,
    "args": {
      "External id": 2466348,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 43
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741519.251, "dur": 25.512,
    "args": {
      "External id": 2466349,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 44
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741549.963, "dur": 6.716,
    "args": {
      "External id": 2466350,"Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 45
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741554.582, "dur": 1.328,
    "args": {
      "External id": 2466351,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 46
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741557.596, "dur": 23.216,
    "args": {
      "External id": 2466352,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 47
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741591.276, "dur": 139.292,
    "args": {
      "External id": 2466353,"Record function id": 0, "Sequence number": 24740643, "Fwd thread id": 1, "Ev Idx": 48
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741592.636, "dur": 132.969,
    "args": {
      "External id": 2466354,"Sequence number": 24740643, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 49
    }
  },
  {
    "ph": "f", "id": 7, "pid": 1336756, "tid": 1381189, "ts": 1514656741592.636,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741597.626, "dur": 127.620,
    "args": {
      "External id": 2466355,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 50
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741601.667, "dur": 17.899,
    "args": {
      "External id": 2466356,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 51
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741602.883, "dur": 3.020,
    "args": {
      "External id": 2466357,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 52
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741606.743, "dur": 12.586,
    "args": {
      "External id": 2466358,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 53
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741607.746, "dur": 11.211,
    "args": {
      "External id": 2466359,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 54
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741623.213, "dur": 2.504,
    "args": {
      "External id": 2466360,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 55
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741624.773, "dur": 0.703,
    "args": {
      "External id": 2466361,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 56
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741626.611, "dur": 97.869,
    "args": {
      "External id": 2466362,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 57
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741737.277, "dur": 105.108,
    "args": {
      "External id": 2466363,"Record function id": 0, "Sequence number": 24740642, "Fwd thread id": 1, "Ev Idx": 58
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741738.474, "dur": 100.842,
    "args": {
      "External id": 2466364,"Sequence number": 24740642, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 59
    }
  },
  {
    "ph": "f", "id": 8, "pid": 1336756, "tid": 1381189, "ts": 1514656741738.474,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741740.126, "dur": 98.951,
    "args": {
      "External id": 2466365,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 60
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741743.649, "dur": 19.495,
    "args": {
      "External id": 2466366,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 61
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741744.573, "dur": 4.570,
    "args": {
      "External id": 2466367,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 62
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741749.864, "dur": 13.034,
    "args": {
      "External id": 2466368,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 63
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741750.689, "dur": 11.875,
    "args": {
      "External id": 2466369,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 64
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741764.071, "dur": 4.159,
    "args": {
      "External id": 2466370,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 65
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741767.360, "dur": 0.751,
    "args": {
      "External id": 2466371,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 66
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741771.188, "dur": 67.151,
    "args": {
      "External id": 2466372,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 67
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741846.523, "dur": 41.719,
    "args": {
      "External id": 2466373,"Record function id": 0, "Sequence number": 24740641, "Fwd thread id": 1, "Ev Idx": 68
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741848.301, "dur": 1.108,
    "args": {
      "External id": 2466374,"Sequence number": 24740641, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 69
    }
  },
  {
    "ph": "f", "id": 9, "pid": 1336756, "tid": 1381189, "ts": 1514656741848.301,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741851.877, "dur": 32.108,
    "args": {
      "External id": 2466375,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 70
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741854.219, "dur": 29.300,
    "args": {
      "External id": 2466376,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 71
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741864.035, "dur": 0.642,
    "args": {
      "External id": 2466377,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 72
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741893.582, "dur": 2237.438,
    "args": {
      "External id": 2466378,"Record function id": 0, "Sequence number": 24740639, "Fwd thread id": 1, "Ev Idx": 73
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741894.863, "dur": 2198.455,
    "args": {
      "External id": 2466379,"Sequence number": 24740639, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 74
    }
  },
  {
    "ph": "f", "id": 10, "pid": 1336756, "tid": 1381189, "ts": 1514656741894.863,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741937.096, "dur": 2.722,
    "args": {
      "External id": 2466380,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 75
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741942.229, "dur": 2037.077,
    "args": {
      "External id": 2466381,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 76
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741944.354, "dur": 2034.687,
    "args": {
      "External id": 2466382,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 77
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741946.975, "dur": 5.347,
    "args": {
      "External id": 2466383,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 78
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656741953.174, "dur": 2024.755,
    "args": {
      "External id": 2466384,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 79
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744004.660, "dur": 0.508,
    "args": {
      "External id": 2466385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 80
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744008.794, "dur": 2.673,
    "args": {
      "External id": 2466386,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 81
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744010.161, "dur": 1.195,
    "args": {
      "External id": 2466387,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 82
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744015.676, "dur": 22.750,
    "args": {
      "External id": 2466388,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 83
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744043.673, "dur": 42.266,
    "args": {
      "External id": 2466389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 84
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744044.918, "dur": 40.829,
    "args": {
      "External id": 2466390,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 85
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744046.246, "dur": 39.153,
    "args": {
      "External id": 2466391,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 86
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744103.822, "dur": 22.429,
    "args": {
      "External id": 2466392,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 87
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744138.371, "dur": 15.142,
    "args": {
      "External id": 2466393,"Record function id": 0, "Sequence number": 24740638, "Fwd thread id": 1, "Ev Idx": 88
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744139.763, "dur": 11.437,
    "args": {
      "External id": 2466394,"Sequence number": 24740638, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 89
    }
  },
  {
    "ph": "f", "id": 11, "pid": 1336756, "tid": 1381189, "ts": 1514656744139.763,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744146.053, "dur": 4.938,
    "args": {
      "External id": 2466395,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 90
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744147.254, "dur": 3.566,
    "args": {
      "External id": 2466396,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 91
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744156.947, "dur": 70.919,
    "args": {
      "External id": 2466397,"Record function id": 0, "Sequence number": 24740637, "Fwd thread id": 1, "Ev Idx": 92
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744157.886, "dur": 65.271,
    "args": {
      "External id": 2466398,"Sequence number": 24740637, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 93
    }
  },
  {
    "ph": "f", "id": 12, "pid": 1336756, "tid": 1381189, "ts": 1514656744157.886,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744159.477, "dur": 63.322,
    "args": {
      "External id": 2466399,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 94
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744164.913, "dur": 21.184,
    "args": {
      "External id": 2466400,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 95
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744166.498, "dur": 3.085,
    "args": {
      "External id": 2466401,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 96
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744170.365, "dur": 15.436,
    "args": {
      "External id": 2466402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 97
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744171.713, "dur": 13.653,
    "args": {
      "External id": 2466403,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 98
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744187.501, "dur": 5.429,
    "args": {
      "External id": 2466404,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 99
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744189.225, "dur": 3.498,
    "args": {
      "External id": 2466405,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744194.065, "dur": 28.059,
    "args": {
      "External id": 2466406,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744233.760, "dur": 56.609,
    "args": {
      "External id": 2466407,"Record function id": 0, "Sequence number": 24740636, "Fwd thread id": 1, "Ev Idx": 102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744234.872, "dur": 51.896,
    "args": {
      "External id": 2466408,"Sequence number": 24740636, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 103
    }
  },
  {
    "ph": "f", "id": 13, "pid": 1336756, "tid": 1381189, "ts": 1514656744234.872,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744239.132, "dur": 47.336,
    "args": {
      "External id": 2466409,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "2"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744240.643, "dur": 20.220,
    "args": {
      "External id": 2466410,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744241.600, "dur": 2.248,
    "args": {
      "External id": 2466411,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744244.309, "dur": 16.263,
    "args": {
      "External id": 2466412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744244.917, "dur": 15.305,
    "args": {
      "External id": 2466413,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744262.039, "dur": 6.574,
    "args": {
      "External id": 2466414,"Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744266.934, "dur": 1.142,
    "args": {
      "External id": 2466415,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744269.446, "dur": 16.531,
    "args": {
      "External id": 2466416,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744293.947, "dur": 97.710,
    "args": {
      "External id": 2466417,"Record function id": 0, "Sequence number": 24740635, "Fwd thread id": 1, "Ev Idx": 112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744297.740, "dur": 90.113,
    "args": {
      "External id": 2466418,"Sequence number": 24740635, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 113
    }
  },
  {
    "ph": "f", "id": 14, "pid": 1336756, "tid": 1381189, "ts": 1514656744297.740,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744299.476, "dur": 88.048,
    "args": {
      "External id": 2466419,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744300.603, "dur": 16.447,
    "args": {
      "External id": 2466420,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744301.586, "dur": 2.033,
    "args": {
      "External id": 2466421,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744304.528, "dur": 12.265,
    "args": {
      "External id": 2466422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744305.326, "dur": 11.100,
    "args": {
      "External id": 2466423,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744318.139, "dur": 4.815,
    "args": {
      "External id": 2466424,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744321.915, "dur": 0.856,
    "args": {
      "External id": 2466425,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744323.792, "dur": 63.153,
    "args": {
      "External id": 2466426,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744395.561, "dur": 95.246,
    "args": {
      "External id": 2466427,"Record function id": 0, "Sequence number": 24740634, "Fwd thread id": 1, "Ev Idx": 122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744398.621, "dur": 74.133,
    "args": {
      "External id": 2466428,"Sequence number": 24740634, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 123
    }
  },
  {
    "ph": "f", "id": 15, "pid": 1336756, "tid": 1381189, "ts": 1514656744398.621,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744401.830, "dur": 70.645,
    "args": {
      "External id": 2466429,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744402.974, "dur": 15.964,
    "args": {
      "External id": 2466430,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744403.620, "dur": 1.718,
    "args": {
      "External id": 2466431,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744405.811, "dur": 12.881,
    "args": {
      "External id": 2466432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744407.102, "dur": 11.282,
    "args": {
      "External id": 2466433,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744419.850, "dur": 2.053,
    "args": {
      "External id": 2466434,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744421.127, "dur": 0.646,
    "args": {
      "External id": 2466435,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744422.812, "dur": 49.006,
    "args": {
      "External id": 2466436,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744476.163, "dur": 13.412,
    "args": {
      "External id": 2466437,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744494.590, "dur": 51.835,
    "args": {
      "External id": 2466438,"Record function id": 0, "Sequence number": 24740633, "Fwd thread id": 1, "Ev Idx": 133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744497.915, "dur": 0.927,
    "args": {
      "External id": 2466439,"Sequence number": 24740633, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 134
    }
  },
  {
    "ph": "f", "id": 16, "pid": 1336756, "tid": 1381189, "ts": 1514656744497.915,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744500.871, "dur": 42.006,
    "args": {
      "External id": 2466440,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744502.610, "dur": 39.565,
    "args": {
      "External id": 2466441,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744509.093, "dur": 2.433,
    "args": {
      "External id": 2466442,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744553.254, "dur": 3091.472,
    "args": {
      "External id": 2466443,"Record function id": 0, "Sequence number": 24740631, "Fwd thread id": 1, "Ev Idx": 138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744557.053, "dur": 3061.332,
    "args": {
      "External id": 2466444,"Sequence number": 24740631, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 139
    }
  },
  {
    "ph": "f", "id": 17, "pid": 1336756, "tid": 1381189, "ts": 1514656744557.053,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744587.279, "dur": 2.325,
    "args": {
      "External id": 2466445,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744591.672, "dur": 2925.468,
    "args": {
      "External id": 2466446,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744592.895, "dur": 2923.954,
    "args": {
      "External id": 2466447,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744595.312, "dur": 5.853,
    "args": {
      "External id": 2466448,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656744602.268, "dur": 2913.580,
    "args": {
      "External id": 2466449,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747520.379, "dur": 0.340,
    "args": {
      "External id": 2466450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747521.827, "dur": 2.476,
    "args": {
      "External id": 2466451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747523.229, "dur": 0.960,
    "args": {
      "External id": 2466452,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747539.122, "dur": 22.488,
    "args": {
      "External id": 2466453,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747572.640, "dur": 39.099,
    "args": {
      "External id": 2466454,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747573.745, "dur": 37.777,
    "args": {
      "External id": 2466455,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747574.951, "dur": 36.189,
    "args": {
      "External id": 2466456,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747626.967, "dur": 14.458,
    "args": {
      "External id": 2466457,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747654.625, "dur": 10.705,
    "args": {
      "External id": 2466458,"Record function id": 0, "Sequence number": 24740630, "Fwd thread id": 1, "Ev Idx": 153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747656.054, "dur": 6.712,
    "args": {
      "External id": 2466459,"Sequence number": 24740630, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 154
    }
  },
  {
    "ph": "f", "id": 18, "pid": 1336756, "tid": 1381189, "ts": 1514656747656.054,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747658.275, "dur": 4.287,
    "args": {
      "External id": 2466460,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747659.393, "dur": 3.025,
    "args": {
      "External id": 2466461,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747668.715, "dur": 70.029,
    "args": {
      "External id": 2466462,"Record function id": 0, "Sequence number": 24740629, "Fwd thread id": 1, "Ev Idx": 157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747669.920, "dur": 63.624,
    "args": {
      "External id": 2466463,"Sequence number": 24740629, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 158
    }
  },
  {
    "ph": "f", "id": 19, "pid": 1336756, "tid": 1381189, "ts": 1514656747669.920,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747674.934, "dur": 58.229,
    "args": {
      "External id": 2466464,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747680.090, "dur": 18.398,
    "args": {
      "External id": 2466465,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747681.114, "dur": 2.527,
    "args": {
      "External id": 2466466,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747684.368, "dur": 13.832,
    "args": {
      "External id": 2466467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747685.794, "dur": 11.987,
    "args": {
      "External id": 2466468,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747699.627, "dur": 5.891,
    "args": {
      "External id": 2466469,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747703.973, "dur": 1.380,
    "args": {
      "External id": 2466470,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747706.863, "dur": 25.522,
    "args": {
      "External id": 2466471,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747742.571, "dur": 52.404,
    "args": {
      "External id": 2466472,"Record function id": 0, "Sequence number": 24740628, "Fwd thread id": 1, "Ev Idx": 167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747743.557, "dur": 48.267,
    "args": {
      "External id": 2466473,"Sequence number": 24740628, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 168
    }
  },
  {
    "ph": "f", "id": 20, "pid": 1336756, "tid": 1381189, "ts": 1514656747743.557,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747744.865, "dur": 46.639,
    "args": {
      "External id": 2466474,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747746.317, "dur": 19.546,
    "args": {
      "External id": 2466475,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747749.660, "dur": 2.044,
    "args": {
      "External id": 2466476,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747752.255, "dur": 13.334,
    "args": {
      "External id": 2466477,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747752.985, "dur": 12.288,
    "args": {
      "External id": 2466478,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747766.945, "dur": 8.362,
    "args": {
      "External id": 2466479,"Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747771.690, "dur": 3.213,
    "args": {
      "External id": 2466480,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747776.070, "dur": 14.993,
    "args": {
      "External id": 2466481,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747798.595, "dur": 98.588,
    "args": {
      "External id": 2466482,"Record function id": 0, "Sequence number": 24740627, "Fwd thread id": 1, "Ev Idx": 177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747799.669, "dur": 94.489,
    "args": {
      "External id": 2466483,"Sequence number": 24740627, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 178
    }
  },
  {
    "ph": "f", "id": 21, "pid": 1336756, "tid": 1381189, "ts": 1514656747799.669,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747801.381, "dur": 92.412,
    "args": {
      "External id": 2466484,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747802.372, "dur": 22.497,
    "args": {
      "External id": 2466485,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747803.573, "dur": 2.171,
    "args": {
      "External id": 2466486,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747808.731, "dur": 15.793,
    "args": {
      "External id": 2466487,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747809.285, "dur": 14.874,
    "args": {
      "External id": 2466488,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747825.895, "dur": 3.978,
    "args": {
      "External id": 2466489,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747829.102, "dur": 0.641,
    "args": {
      "External id": 2466490,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747830.900, "dur": 62.201,
    "args": {
      "External id": 2466491,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747900.883, "dur": 132.967,
    "args": {
      "External id": 2466492,"Record function id": 0, "Sequence number": 24740626, "Fwd thread id": 1, "Ev Idx": 187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747902.096, "dur": 75.439,
    "args": {
      "External id": 2466493,"Sequence number": 24740626, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 188
    }
  },
  {
    "ph": "f", "id": 22, "pid": 1336756, "tid": 1381189, "ts": 1514656747902.096,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747905.963, "dur": 71.245,
    "args": {
      "External id": 2466494,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747906.758, "dur": 17.744,
    "args": {
      "External id": 2466495,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747907.249, "dur": 1.599,
    "args": {
      "External id": 2466496,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747909.423, "dur": 14.819,
    "args": {
      "External id": 2466497,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747913.057, "dur": 10.879,
    "args": {
      "External id": 2466498,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747925.354, "dur": 4.109,
    "args": {
      "External id": 2466499,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747928.549, "dur": 0.747,
    "args": {
      "External id": 2466500,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747930.308, "dur": 46.360,
    "args": {
      "External id": 2466501,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656747980.538, "dur": 51.454,
    "args": {
      "External id": 2466502,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748040.103, "dur": 34.016,
    "args": {
      "External id": 2466503,"Record function id": 0, "Sequence number": 24740625, "Fwd thread id": 1, "Ev Idx": 198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748041.757, "dur": 1.459,
    "args": {
      "External id": 2466504,"Sequence number": 24740625, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 199
    }
  },
  {
    "ph": "f", "id": 23, "pid": 1336756, "tid": 1381189, "ts": 1514656748041.757,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748045.292, "dur": 24.056,
    "args": {
      "External id": 2466505,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748047.002, "dur": 21.869,
    "args": {
      "External id": 2466506,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748052.015, "dur": 0.930,
    "args": {
      "External id": 2466507,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748078.228, "dur": 3063.126,
    "args": {
      "External id": 2466508,"Record function id": 0, "Sequence number": 24740624, "Fwd thread id": 1, "Ev Idx": 203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748091.670, "dur": 3019.296,
    "args": {
      "External id": 2466509,"Sequence number": 24740624, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 204
    }
  },
  {
    "ph": "f", "id": 24, "pid": 1336756, "tid": 1381189, "ts": 1514656748091.670,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748119.867, "dur": 2.620,
    "args": {
      "External id": 2466510,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748124.559, "dur": 2906.744,
    "args": {
      "External id": 2466511,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748125.714, "dur": 2905.329,
    "args": {
      "External id": 2466512,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748130.639, "dur": 3.082,
    "args": {
      "External id": 2466513,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656748134.748, "dur": 2895.522,
    "args": {
      "External id": 2466514,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751034.396, "dur": 0.274,
    "args": {
      "External id": 2466515,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751035.608, "dur": 1.880,
    "args": {
      "External id": 2466516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751036.738, "dur": 0.635,
    "args": {
      "External id": 2466517,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751040.561, "dur": 21.650,
    "args": {
      "External id": 2466518,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751066.571, "dur": 38.432,
    "args": {
      "External id": 2466519,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751067.334, "dur": 37.479,
    "args": {
      "External id": 2466520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751068.204, "dur": 36.231,
    "args": {
      "External id": 2466521,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751120.677, "dur": 15.582,
    "args": {
      "External id": 2466522,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751153.111, "dur": 14.206,
    "args": {
      "External id": 2466523,"Record function id": 0, "Ev Idx": 218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751155.928, "dur": 9.958,
    "args": {
      "External id": 2466524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751159.870, "dur": 4.960,
    "args": {
      "External id": 2466525,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751160.867, "dur": 3.847,
    "args": {
      "External id": 2466526,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751170.969, "dur": 11.879,
    "args": {
      "External id": 2466527,"Record function id": 0, "Sequence number": 24740623, "Fwd thread id": 1, "Ev Idx": 222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751172.219, "dur": 7.513,
    "args": {
      "External id": 2466528,"Sequence number": 24740623, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 223
    }
  },
  {
    "ph": "f", "id": 25, "pid": 1336756, "tid": 1381189, "ts": 1514656751172.219,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751174.065, "dur": 5.472,
    "args": {
      "External id": 2466529,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751177.279, "dur": 2.163,
    "args": {
      "External id": 2466530,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751186.242, "dur": 69.605,
    "args": {
      "External id": 2466531,"Record function id": 0, "Sequence number": 24740622, "Fwd thread id": 1, "Ev Idx": 226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751187.096, "dur": 63.496,
    "args": {
      "External id": 2466532,"Sequence number": 24740622, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 227
    }
  },
  {
    "ph": "f", "id": 26, "pid": 1336756, "tid": 1381189, "ts": 1514656751187.096,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751189.075, "dur": 61.200,
    "args": {
      "External id": 2466533,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751193.838, "dur": 22.316,
    "args": {
      "External id": 2466534,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751195.416, "dur": 2.858,
    "args": {
      "External id": 2466535,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751199.108, "dur": 16.773,
    "args": {
      "External id": 2466536,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751200.552, "dur": 14.908,
    "args": {
      "External id": 2466537,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751217.418, "dur": 5.169,
    "args": {
      "External id": 2466538,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751221.415, "dur": 0.906,
    "args": {
      "External id": 2466539,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751223.401, "dur": 25.717,
    "args": {
      "External id": 2466540,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751261.809, "dur": 53.100,
    "args": {
      "External id": 2466541,"Record function id": 0, "Sequence number": 24740621, "Fwd thread id": 1, "Ev Idx": 236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751262.954, "dur": 49.482,
    "args": {
      "External id": 2466542,"Sequence number": 24740621, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 237
    }
  },
  {
    "ph": "f", "id": 27, "pid": 1336756, "tid": 1381189, "ts": 1514656751262.954,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751264.582, "dur": 47.641,
    "args": {
      "External id": 2466543,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751265.945, "dur": 25.866,
    "args": {
      "External id": 2466544,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751273.968, "dur": 2.075,
    "args": {
      "External id": 2466545,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751276.701, "dur": 14.836,
    "args": {
      "External id": 2466546,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751277.623, "dur": 13.527,
    "args": {
      "External id": 2466547,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751292.837, "dur": 3.977,
    "args": {
      "External id": 2466548,"Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751295.311, "dur": 1.036,
    "args": {
      "External id": 2466549,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751297.199, "dur": 14.482,
    "args": {
      "External id": 2466550,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751318.504, "dur": 98.898,
    "args": {
      "External id": 2466551,"Record function id": 0, "Sequence number": 24740620, "Fwd thread id": 1, "Ev Idx": 246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751319.620, "dur": 94.342,
    "args": {
      "External id": 2466552,"Sequence number": 24740620, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 247
    }
  },
  {
    "ph": "f", "id": 28, "pid": 1336756, "tid": 1381189, "ts": 1514656751319.620,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751321.592, "dur": 91.956,
    "args": {
      "External id": 2466553,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751324.541, "dur": 20.649,
    "args": {
      "External id": 2466554,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751325.791, "dur": 1.996,
    "args": {
      "External id": 2466555,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751331.207, "dur": 13.751,
    "args": {
      "External id": 2466556,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751332.066, "dur": 12.533,
    "args": {
      "External id": 2466557,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751346.175, "dur": 4.000,
    "args": {
      "External id": 2466558,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751347.516, "dur": 2.531,
    "args": {
      "External id": 2466559,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751350.734, "dur": 62.100,
    "args": {
      "External id": 2466560,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751421.236, "dur": 102.376,
    "args": {
      "External id": 2466561,"Record function id": 0, "Sequence number": 24740619, "Fwd thread id": 1, "Ev Idx": 256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751422.296, "dur": 83.857,
    "args": {
      "External id": 2466562,"Sequence number": 24740619, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 257
    }
  },
  {
    "ph": "f", "id": 29, "pid": 1336756, "tid": 1381189, "ts": 1514656751422.296,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751423.808, "dur": 82.056,
    "args": {
      "External id": 2466563,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751424.984, "dur": 28.637,
    "args": {
      "External id": 2466564,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751430.718, "dur": 1.579,
    "args": {
      "External id": 2466565,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751435.927, "dur": 17.439,
    "args": {
      "External id": 2466566,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751441.642, "dur": 11.334,
    "args": {
      "External id": 2466567,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751454.679, "dur": 1.897,
    "args": {
      "External id": 2466568,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751455.727, "dur": 0.648,
    "args": {
      "External id": 2466569,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751457.474, "dur": 47.748,
    "args": {
      "External id": 2466570,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751509.723, "dur": 12.188,
    "args": {
      "External id": 2466571,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751543.035, "dur": 375.864,
    "args": {
      "External id": 2466572,"Record function id": 0, "Sequence number": 24740618, "Fwd thread id": 1, "Ev Idx": 267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751545.337, "dur": 364.564,
    "args": {
      "External id": 2466573,"Sequence number": 24740618, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 268
    }
  },
  {
    "ph": "f", "id": 30, "pid": 1336756, "tid": 1381189, "ts": 1514656751545.337,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751730.116, "dur": 47.408,
    "args": {
      "External id": 2466574,"kernel_hash": "coj6y6gy5rvprivwlbvxp47673cukqlcygvp5cpbrgwb4yp2fdt6", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "262144", "2048", "1", "1986", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/oj/coj6y6gy5rvprivwlbvxp47673cukqlcygvp5cpbrgwb4yp2fdt6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [2048], [262144, 2048], [262144, 2048], [132, 2048], [262144], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751816.519, "dur": 31.347,
    "args": {
      "External id": 2466575,"kernel_hash": "crljzlkpbfozvboy2akosvc2vfrg5cb56xolbkj7jk22iz5mrjtx", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/rl/crljzlkpbfozvboy2akosvc2vfrg5cb56xolbkj7jk22iz5mrjtx.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751865.587, "dur": 19.295,
    "args": {
      "External id": 2466576,"kernel_hash": "cwikyxkzux4tjcfntxriioajhoy22fwo4vn7g7obctqs5aqrtd3v", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/wi/cwikyxkzux4tjcfntxriioajhoy22fwo4vn7g7obctqs5aqrtd3v.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751929.591, "dur": 13.522,
    "args": {
      "External id": 2466577,"Record function id": 0, "Ev Idx": 272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751931.408, "dur": 10.921,
    "args": {
      "External id": 2466578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751934.646, "dur": 6.775,
    "args": {
      "External id": 2466579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751938.394, "dur": 2.925,
    "args": {
      "External id": 2466580,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: StackBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751949.092, "dur": 70.589,
    "args": {
      "External id": 2466581,"Record function id": 0, "Sequence number": 24740617, "Fwd thread id": 1, "Ev Idx": 276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "StackBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751950.165, "dur": 25.979,
    "args": {
      "External id": 2466582,"Sequence number": 24740617, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 277
    }
  },
  {
    "ph": "f", "id": 31, "pid": 1336756, "tid": 1381189, "ts": 1514656751950.165,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751952.342, "dur": 8.003,
    "args": {
      "External id": 2466583,"Record function id": 0, "Concrete Inputs": ["", "-2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751957.412, "dur": 1.298,
    "args": {
      "External id": 2466584,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751960.871, "dur": 6.872,
    "args": {
      "External id": 2466585,"Record function id": 0, "Concrete Inputs": ["", "-2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751966.675, "dur": 0.446,
    "args": {
      "External id": 2466586,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751968.224, "dur": 2.063,
    "args": {
      "External id": 2466587,"Record function id": 0, "Concrete Inputs": ["", "-2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751969.337, "dur": 0.319,
    "args": {
      "External id": 2466588,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751970.792, "dur": 4.683,
    "args": {
      "External id": 2466589,"Record function id": 0, "Concrete Inputs": ["", "-2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656751974.154, "dur": 0.662,
    "args": {
      "External id": 2466590,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752026.129, "dur": 6.744,
    "args": {
      "External id": 2466591,"Record function id": 0, "Sequence number": 24740616, "Fwd thread id": 1, "Ev Idx": 286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752027.441, "dur": 1.234,
    "args": {
      "External id": 2466592,"Sequence number": 24740616, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 287
    }
  },
  {
    "ph": "f", "id": 32, "pid": 1336756, "tid": 1381189, "ts": 1514656752027.441,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752037.382, "dur": 557.747,
    "args": {
      "External id": 2466593,"Record function id": 0, "Sequence number": 24740615, "Fwd thread id": 1, "Ev Idx": 288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752038.495, "dur": 538.771,
    "args": {
      "External id": 2466594,"Sequence number": 24740615, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 289
    }
  },
  {
    "ph": "f", "id": 33, "pid": 1336756, "tid": 1381189, "ts": 1514656752038.495,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752082.583, "dur": 12.915,
    "args": {
      "External id": 2466595,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752090.688, "dur": 4.412,
    "args": {
      "External id": 2466596,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752102.264, "dur": 7.254,
    "args": {
      "External id": 2466597,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752106.320, "dur": 2.314,
    "args": {
      "External id": 2466598,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752107.740, "dur": 0.650,
    "args": {
      "External id": 2466599,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752117.520, "dur": 130.709,
    "args": {
      "External id": 2466600,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752118.697, "dur": 7.416,
    "args": {
      "External id": 2466601,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752119.390, "dur": 5.905,
    "args": {
      "External id": 2466602,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752122.340, "dur": 2.890,
    "args": {
      "External id": 2466603,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752128.522, "dur": 118.981,
    "args": {
      "External id": 2466604,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752131.038, "dur": 115.236,
    "args": {
      "External id": 2466605,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752253.112, "dur": 6.698,
    "args": {
      "External id": 2466606,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752257.087, "dur": 2.565,
    "args": {
      "External id": 2466607,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752295.765, "dur": 4.849,
    "args": {
      "External id": 2466608,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752301.555, "dur": 3.822,
    "args": {
      "External id": 2466609,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752308.445, "dur": 1.498,
    "args": {
      "External id": 2466610,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752348.256, "dur": 2.316,
    "args": {
      "External id": 2466611,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752349.058, "dur": 1.347,
    "args": {
      "External id": 2466612,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752383.144, "dur": 168.703,
    "args": {
      "External id": 2466613,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752390.223, "dur": 8.626,
    "args": {
      "External id": 2466614,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752395.640, "dur": 0.839,
    "args": {
      "External id": 2466615,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752400.649, "dur": 8.454,
    "args": {
      "External id": 2466616,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752405.789, "dur": 2.688,
    "args": {
      "External id": 2466617,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752410.547, "dur": 4.822,
    "args": {
      "External id": 2466618,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752414.232, "dur": 0.752,
    "args": {
      "External id": 2466619,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752415.955, "dur": 4.482,
    "args": {
      "External id": 2466620,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752419.177, "dur": 0.837,
    "args": {
      "External id": 2466621,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752424.442, "dur": 4.236,
    "args": {
      "External id": 2466622,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752428.013, "dur": 0.381,
    "args": {
      "External id": 2466623,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752431.707, "dur": 6.936,
    "args": {
      "External id": 2466624,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752436.838, "dur": 1.604,
    "args": {
      "External id": 2466625,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752439.410, "dur": 3.717,
    "args": {
      "External id": 2466626,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752442.331, "dur": 0.530,
    "args": {
      "External id": 2466627,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752443.672, "dur": 2.162,
    "args": {
      "External id": 2466628,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752444.468, "dur": 1.281,
    "args": {
      "External id": 2466629,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752447.856, "dur": 69.850,
    "args": {
      "External id": 2466630,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752519.278, "dur": 3.199,
    "args": {
      "External id": 2466631,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752525.777, "dur": 21.029,
    "args": {
      "External id": 2466632,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752544.832, "dur": 0.883,
    "args": {
      "External id": 2466633,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752549.102, "dur": 1.381,
    "args": {
      "External id": 2466634,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752608.514, "dur": 11.285,
    "args": {
      "External id": 2466635,"Record function id": 0, "Ev Idx": 330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752610.732, "dur": 8.184,
    "args": {
      "External id": 2466636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752613.683, "dur": 4.350,
    "args": {
      "External id": 2466637,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752615.225, "dur": 2.717,
    "args": {
      "External id": 2466638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752623.843, "dur": 9.535,
    "args": {
      "External id": 2466639,"Record function id": 0, "Sequence number": 24740614, "Fwd thread id": 1, "Ev Idx": 334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752625.080, "dur": 5.760,
    "args": {
      "External id": 2466640,"Sequence number": 24740614, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 335
    }
  },
  {
    "ph": "f", "id": 34, "pid": 1336756, "tid": 1381189, "ts": 1514656752625.080,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752628.891, "dur": 1.764,
    "args": {
      "External id": 2466641,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752629.845, "dur": 0.718,
    "args": {
      "External id": 2466642,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752637.008, "dur": 184.761,
    "args": {
      "External id": 2466643,"Record function id": 0, "Sequence number": 24740613, "Fwd thread id": 1, "Ev Idx": 338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752637.875, "dur": 174.667,
    "args": {
      "External id": 2466644,"Sequence number": 24740613, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 339
    }
  },
  {
    "ph": "f", "id": 35, "pid": 1336756, "tid": 1381189, "ts": 1514656752637.875,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752644.977, "dur": 4.514,
    "args": {
      "External id": 2466645,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752646.407, "dur": 2.525,
    "args": {
      "External id": 2466646,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752647.979, "dur": 0.787,
    "args": {
      "External id": 2466647,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752651.017, "dur": 66.203,
    "args": {
      "External id": 2466648,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752718.626, "dur": 6.158,
    "args": {
      "External id": 2466649,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752719.291, "dur": 4.941,
    "args": {
      "External id": 2466650,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752722.927, "dur": 1.118,
    "args": {
      "External id": 2466651,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752726.660, "dur": 5.747,
    "args": {
      "External id": 2466652,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752727.845, "dur": 4.096,
    "args": {
      "External id": 2466653,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752731.357, "dur": 0.517,
    "args": {
      "External id": 2466654,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752733.002, "dur": 78.707,
    "args": {
      "External id": 2466655,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752826.455, "dur": 6.142,
    "args": {
      "External id": 2466656,"Record function id": 0, "Sequence number": 24740612, "Fwd thread id": 1, "Ev Idx": 351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752827.677, "dur": 3.513,
    "args": {
      "External id": 2466657,"Sequence number": 24740612, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 352
    }
  },
  {
    "ph": "f", "id": 36, "pid": 1336756, "tid": 1381189, "ts": 1514656752827.677,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752829.139, "dur": 1.893,
    "args": {
      "External id": 2466658,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752829.790, "dur": 1.143,
    "args": {
      "External id": 2466659,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752835.977, "dur": 11.966,
    "args": {
      "External id": 2466660,"Record function id": 0, "Sequence number": 24740611, "Fwd thread id": 1, "Ev Idx": 355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752838.624, "dur": 6.288,
    "args": {
      "External id": 2466661,"Sequence number": 24740611, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 356
    }
  },
  {
    "ph": "f", "id": 37, "pid": 1336756, "tid": 1381189, "ts": 1514656752838.624,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752839.853, "dur": 4.841,
    "args": {
      "External id": 2466662,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752840.567, "dur": 3.649,
    "args": {
      "External id": 2466663,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752843.617, "dur": 0.497,
    "args": {
      "External id": 2466664,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752851.762, "dur": 6.033,
    "args": {
      "External id": 2466665,"Record function id": 0, "Ev Idx": 360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752853.362, "dur": 3.957,
    "args": {
      "External id": 2466666,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752854.601, "dur": 2.471,
    "args": {
      "External id": 2466667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752855.458, "dur": 1.503,
    "args": {
      "External id": 2466668,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752861.094, "dur": 8.910,
    "args": {
      "External id": 2466669,"Record function id": 0, "Sequence number": 24740610, "Fwd thread id": 1, "Ev Idx": 364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752862.164, "dur": 5.704,
    "args": {
      "External id": 2466670,"Sequence number": 24740610, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 365
    }
  },
  {
    "ph": "f", "id": 38, "pid": 1336756, "tid": 1381189, "ts": 1514656752862.164,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752863.620, "dur": 4.107,
    "args": {
      "External id": 2466671,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752866.727, "dur": 0.908,
    "args": {
      "External id": 2466672,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752873.147, "dur": 166.615,
    "args": {
      "External id": 2466673,"Record function id": 0, "Sequence number": 24740609, "Fwd thread id": 1, "Ev Idx": 368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752873.858, "dur": 153.074,
    "args": {
      "External id": 2466674,"Sequence number": 24740609, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 369
    }
  },
  {
    "ph": "f", "id": 39, "pid": 1336756, "tid": 1381189, "ts": 1514656752873.858,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752876.355, "dur": 2.386,
    "args": {
      "External id": 2466675,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752877.025, "dur": 1.314,
    "args": {
      "External id": 2466676,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752877.646, "dur": 0.593,
    "args": {
      "External id": 2466677,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752881.665, "dur": 44.899,
    "args": {
      "External id": 2466678,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752927.591, "dur": 4.768,
    "args": {
      "External id": 2466679,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752928.077, "dur": 3.773,
    "args": {
      "External id": 2466680,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752930.785, "dur": 0.935,
    "args": {
      "External id": 2466681,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752933.550, "dur": 8.235,
    "args": {
      "External id": 2466682,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752934.870, "dur": 6.469,
    "args": {
      "External id": 2466683,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752939.075, "dur": 2.181,
    "args": {
      "External id": 2466684,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656752942.654, "dur": 82.791,
    "args": {
      "External id": 2466685,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753049.515, "dur": 41.931,
    "args": {
      "External id": 2466686,"Record function id": 0, "Sequence number": 24740608, "Fwd thread id": 1, "Ev Idx": 381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753051.009, "dur": 6.981,
    "args": {
      "External id": 2466687,"Sequence number": 24740608, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 382
    }
  },
  {
    "ph": "f", "id": 40, "pid": 1336756, "tid": 1381189, "ts": 1514656753051.009,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753053.045, "dur": 4.799,
    "args": {
      "External id": 2466688,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753056.170, "dur": 1.578,
    "args": {
      "External id": 2466689,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753061.930, "dur": 26.352,
    "args": {
      "External id": 2466690,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753095.314, "dur": 11.347,
    "args": {
      "External id": 2466691,"Record function id": 0, "Sequence number": 24740607, "Fwd thread id": 1, "Ev Idx": 386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753096.589, "dur": 7.539,
    "args": {
      "External id": 2466692,"Sequence number": 24740607, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 387
    }
  },
  {
    "ph": "f", "id": 41, "pid": 1336756, "tid": 1381189, "ts": 1514656753096.589,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753097.705, "dur": 6.201,
    "args": {
      "External id": 2466693,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753101.191, "dur": 2.135,
    "args": {
      "External id": 2466694,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753102.595, "dur": 0.614,
    "args": {
      "External id": 2466695,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753110.625, "dur": 6.246,
    "args": {
      "External id": 2466696,"Record function id": 0, "Ev Idx": 391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753112.060, "dur": 4.265,
    "args": {
      "External id": 2466697,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753113.435, "dur": 2.550,
    "args": {
      "External id": 2466698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753113.974, "dur": 1.809,
    "args": {
      "External id": 2466699,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753121.485, "dur": 508.134,
    "args": {
      "External id": 2466700,"Record function id": 0, "Sequence number": 24740606, "Fwd thread id": 1, "Ev Idx": 395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753122.938, "dur": 488.866,
    "args": {
      "External id": 2466701,"Sequence number": 24740606, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 396
    }
  },
  {
    "ph": "f", "id": 42, "pid": 1336756, "tid": 1381189, "ts": 1514656753122.938,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753154.319, "dur": 36.177,
    "args": {
      "External id": 2466702,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753155.922, "dur": 34.329,
    "args": {
      "External id": 2466703,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753158.700, "dur": 6.211,
    "args": {
      "External id": 2466704,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753161.059, "dur": 3.259,
    "args": {
      "External id": 2466705,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753166.392, "dur": 23.297,
    "args": {
      "External id": 2466706,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753202.656, "dur": 4.914,
    "args": {
      "External id": 2466707,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753205.853, "dur": 1.603,
    "args": {
      "External id": 2466708,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753211.805, "dur": 3.723,
    "args": {
      "External id": 2466709,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753212.418, "dur": 3.011,
    "args": {
      "External id": 2466710,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753230.414, "dur": 2.105,
    "args": {
      "External id": 2466711,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753248.291, "dur": 3.279,
    "args": {
      "External id": 2466712,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753473.337, "dur": 2.966,
    "args": {
      "External id": 2466713,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753481.376, "dur": 35.114,
    "args": {
      "External id": 2466714,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753492.221, "dur": 1.088,
    "args": {
      "External id": 2466715,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753522.354, "dur": 50.738,
    "args": {
      "External id": 2466716,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753524.312, "dur": 48.542,
    "args": {
      "External id": 2466717,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753544.866, "dur": 5.147,
    "args": {
      "External id": 2466718,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753551.421, "dur": 20.770,
    "args": {
      "External id": 2466719,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753580.954, "dur": 3.220,
    "args": {
      "External id": 2466720,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753582.572, "dur": 1.470,
    "args": {
      "External id": 2466721,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753591.051, "dur": 5.088,
    "args": {
      "External id": 2466722,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753594.566, "dur": 1.465,
    "args": {
      "External id": 2466723,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753598.364, "dur": 3.660,
    "args": {
      "External id": 2466724,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753599.090, "dur": 2.846,
    "args": {
      "External id": 2466725,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753642.075, "dur": 9.387,
    "args": {
      "External id": 2466726,"Record function id": 0, "Ev Idx": 421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753644.376, "dur": 6.450,
    "args": {
      "External id": 2466727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753646.584, "dur": 3.217,
    "args": {
      "External id": 2466728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753647.772, "dur": 1.760,
    "args": {
      "External id": 2466729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753654.823, "dur": 10.791,
    "args": {
      "External id": 2466730,"Record function id": 0, "Sequence number": 24740605, "Fwd thread id": 1, "Ev Idx": 425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753655.914, "dur": 6.387,
    "args": {
      "External id": 2466731,"Sequence number": 24740605, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 426
    }
  },
  {
    "ph": "f", "id": 43, "pid": 1336756, "tid": 1381189, "ts": 1514656753655.914,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753659.827, "dur": 2.286,
    "args": {
      "External id": 2466732,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753660.735, "dur": 1.229,
    "args": {
      "External id": 2466733,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753671.120, "dur": 144.900,
    "args": {
      "External id": 2466734,"Record function id": 0, "Sequence number": 24740604, "Fwd thread id": 1, "Ev Idx": 429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753672.314, "dur": 137.735,
    "args": {
      "External id": 2466735,"Sequence number": 24740604, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 430
    }
  },
  {
    "ph": "f", "id": 44, "pid": 1336756, "tid": 1381189, "ts": 1514656753672.314,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753675.926, "dur": 4.187,
    "args": {
      "External id": 2466736,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753677.299, "dur": 2.268,
    "args": {
      "External id": 2466737,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753678.566, "dur": 0.841,
    "args": {
      "External id": 2466738,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753681.210, "dur": 61.952,
    "args": {
      "External id": 2466739,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753744.511, "dur": 8.690,
    "args": {
      "External id": 2466740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753747.858, "dur": 4.629,
    "args": {
      "External id": 2466741,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753749.662, "dur": 2.634,
    "args": {
      "External id": 2466742,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753754.870, "dur": 5.013,
    "args": {
      "External id": 2466743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753755.946, "dur": 3.499,
    "args": {
      "External id": 2466744,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753758.935, "dur": 0.426,
    "args": {
      "External id": 2466745,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753760.549, "dur": 48.698,
    "args": {
      "External id": 2466746,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753820.888, "dur": 13.521,
    "args": {
      "External id": 2466747,"Record function id": 0, "Sequence number": 24740603, "Fwd thread id": 1, "Ev Idx": 442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753821.924, "dur": 9.499,
    "args": {
      "External id": 2466748,"Sequence number": 24740603, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 443
    }
  },
  {
    "ph": "f", "id": 45, "pid": 1336756, "tid": 1381189, "ts": 1514656753821.924,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753826.374, "dur": 4.892,
    "args": {
      "External id": 2466749,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753827.326, "dur": 3.798,
    "args": {
      "External id": 2466750,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753837.878, "dur": 8.923,
    "args": {
      "External id": 2466751,"Record function id": 0, "Sequence number": 24740602, "Fwd thread id": 1, "Ev Idx": 446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753839.085, "dur": 6.204,
    "args": {
      "External id": 2466752,"Sequence number": 24740602, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 447
    }
  },
  {
    "ph": "f", "id": 46, "pid": 1336756, "tid": 1381189, "ts": 1514656753839.085,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753840.126, "dur": 4.967,
    "args": {
      "External id": 2466753,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753840.929, "dur": 3.688,
    "args": {
      "External id": 2466754,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753843.953, "dur": 0.525,
    "args": {
      "External id": 2466755,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753851.025, "dur": 8.139,
    "args": {
      "External id": 2466756,"Record function id": 0, "Ev Idx": 451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753852.211, "dur": 6.464,
    "args": {
      "External id": 2466757,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753853.420, "dur": 4.842,
    "args": {
      "External id": 2466758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753856.912, "dur": 1.226,
    "args": {
      "External id": 2466759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753862.073, "dur": 7.452,
    "args": {
      "External id": 2466760,"Record function id": 0, "Sequence number": 24740601, "Fwd thread id": 1, "Ev Idx": 455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753863.106, "dur": 3.922,
    "args": {
      "External id": 2466761,"Sequence number": 24740601, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 456
    }
  },
  {
    "ph": "f", "id": 47, "pid": 1336756, "tid": 1381189, "ts": 1514656753863.106,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753864.547, "dur": 2.324,
    "args": {
      "External id": 2466762,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753865.562, "dur": 1.165,
    "args": {
      "External id": 2466763,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753874.384, "dur": 383.808,
    "args": {
      "External id": 2466764,"Record function id": 0, "Sequence number": 24740600, "Fwd thread id": 1, "Ev Idx": 459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753875.916, "dur": 358.685,
    "args": {
      "External id": 2466765,"Sequence number": 24740600, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 460
    }
  },
  {
    "ph": "f", "id": 48, "pid": 1336756, "tid": 1381189, "ts": 1514656753875.916,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753894.690, "dur": 10.401,
    "args": {
      "External id": 2466766,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753901.433, "dur": 3.220,
    "args": {
      "External id": 2466767,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753907.263, "dur": 3.319,
    "args": {
      "External id": 2466768,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753908.457, "dur": 1.875,
    "args": {
      "External id": 2466769,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753911.995, "dur": 6.198,
    "args": {
      "External id": 2466770,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753915.057, "dur": 2.931,
    "args": {
      "External id": 2466771,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656753949.240, "dur": 257.828,
    "args": {
      "External id": 2466772,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754090.276, "dur": 4.920,
    "args": {
      "External id": 2466773,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754096.827, "dur": 4.420,
    "args": {
      "External id": 2466774,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754220.749, "dur": 4.331,
    "args": {
      "External id": 2466775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754228.167, "dur": 0.905,
    "args": {
      "External id": 2466776,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754231.026, "dur": 0.712,
    "args": {
      "External id": 2466777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754268.228, "dur": 248.677,
    "args": {
      "External id": 2466778,"Record function id": 0, "Sequence number": 24740599, "Fwd thread id": 1, "Ev Idx": 473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754269.691, "dur": 239.165,
    "args": {
      "External id": 2466779,"Sequence number": 24740599, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 474
    }
  },
  {
    "ph": "f", "id": 49, "pid": 1336756, "tid": 1381189, "ts": 1514656754269.691,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754292.932, "dur": 52.125,
    "args": {
      "External id": 2466780,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754299.901, "dur": 2.976,
    "args": {
      "External id": 2466781,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754304.443, "dur": 40.113,
    "args": {
      "External id": 2466782,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754355.377, "dur": 3.570,
    "args": {
      "External id": 2466783,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754356.774, "dur": 1.860,
    "args": {
      "External id": 2466784,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754524.043, "dur": 184.357,
    "args": {
      "External id": 2466785,"Record function id": 0, "Sequence number": 24740598, "Fwd thread id": 1, "Ev Idx": 480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754525.590, "dur": 175.858,
    "args": {
      "External id": 2466786,"Sequence number": 24740598, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 481
    }
  },
  {
    "ph": "f", "id": 50, "pid": 1336756, "tid": 1381189, "ts": 1514656754525.590,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754554.360, "dur": 41.493,
    "args": {
      "External id": 2466787,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754559.419, "dur": 3.858,
    "args": {
      "External id": 2466788,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754564.296, "dur": 30.988,
    "args": {
      "External id": 2466789,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754603.063, "dur": 3.999,
    "args": {
      "External id": 2466790,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754604.389, "dur": 2.319,
    "args": {
      "External id": 2466791,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754718.956, "dur": 15.349,
    "args": {
      "External id": 2466792,"Record function id": 0, "Sequence number": 24740597, "Fwd thread id": 1, "Ev Idx": 487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754720.375, "dur": 10.525,
    "args": {
      "External id": 2466793,"Sequence number": 24740597, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 488
    }
  },
  {
    "ph": "f", "id": 51, "pid": 1336756, "tid": 1381189, "ts": 1514656754720.375,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754722.915, "dur": 7.735,
    "args": {
      "External id": 2466794,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754724.062, "dur": 6.306,
    "args": {
      "External id": 2466795,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754737.920, "dur": 10.554,
    "args": {
      "External id": 2466796,"Record function id": 0, "Sequence number": 24740596, "Fwd thread id": 1, "Ev Idx": 491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754738.701, "dur": 7.005,
    "args": {
      "External id": 2466797,"Sequence number": 24740596, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 492
    }
  },
  {
    "ph": "f", "id": 52, "pid": 1336756, "tid": 1381189, "ts": 1514656754738.701,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754743.445, "dur": 2.079,
    "args": {
      "External id": 2466798,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754744.245, "dur": 1.150,
    "args": {
      "External id": 2466799,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754751.734, "dur": 7.987,
    "args": {
      "External id": 2466800,"Record function id": 0, "Sequence number": 24740595, "Fwd thread id": 1, "Ev Idx": 495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754752.751, "dur": 4.981,
    "args": {
      "External id": 2466801,"Sequence number": 24740595, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 496
    }
  },
  {
    "ph": "f", "id": 53, "pid": 1336756, "tid": 1381189, "ts": 1514656754752.751,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754756.024, "dur": 1.566,
    "args": {
      "External id": 2466802,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754756.537, "dur": 0.955,
    "args": {
      "External id": 2466803,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754765.070, "dur": 7.070,
    "args": {
      "External id": 2466804,"Record function id": 0, "Sequence number": 24740594, "Fwd thread id": 1, "Ev Idx": 499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754766.071, "dur": 3.502,
    "args": {
      "External id": 2466805,"Sequence number": 24740594, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 500
    }
  },
  {
    "ph": "f", "id": 54, "pid": 1336756, "tid": 1381189, "ts": 1514656754766.071,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754767.444, "dur": 1.983,
    "args": {
      "External id": 2466806,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754768.040, "dur": 1.263,
    "args": {
      "External id": 2466807,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754775.596, "dur": 178.726,
    "args": {
      "External id": 2466808,"Record function id": 0, "Sequence number": 24740593, "Fwd thread id": 1, "Ev Idx": 503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754776.961, "dur": 169.897,
    "args": {
      "External id": 2466809,"Sequence number": 24740593, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 504
    }
  },
  {
    "ph": "f", "id": 55, "pid": 1336756, "tid": 1381189, "ts": 1514656754776.961,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754781.132, "dur": 8.706,
    "args": {
      "External id": 2466810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754785.569, "dur": 3.654,
    "args": {
      "External id": 2466811,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754787.494, "dur": 1.468,
    "args": {
      "External id": 2466812,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754791.328, "dur": 67.490,
    "args": {
      "External id": 2466813,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754859.931, "dur": 6.237,
    "args": {
      "External id": 2466814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754861.009, "dur": 4.451,
    "args": {
      "External id": 2466815,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754864.110, "dur": 1.152,
    "args": {
      "External id": 2466816,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754892.147, "dur": 5.836,
    "args": {
      "External id": 2466817,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754895.976, "dur": 1.254,
    "args": {
      "External id": 2466818,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754896.672, "dur": 0.495,
    "args": {
      "External id": 2466819,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754898.867, "dur": 46.960,
    "args": {
      "External id": 2466820,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754959.454, "dur": 7.031,
    "args": {
      "External id": 2466821,"Record function id": 0, "Sequence number": 24740592, "Fwd thread id": 1, "Ev Idx": 516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754960.445, "dur": 3.646,
    "args": {
      "External id": 2466822,"Sequence number": 24740592, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 517
    }
  },
  {
    "ph": "f", "id": 56, "pid": 1336756, "tid": 1381189, "ts": 1514656754960.445,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754961.952, "dur": 1.978,
    "args": {
      "External id": 2466823,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754962.615, "dur": 1.215,
    "args": {
      "External id": 2466824,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754969.539, "dur": 11.105,
    "args": {
      "External id": 2466825,"Record function id": 0, "Sequence number": 24740591, "Fwd thread id": 1, "Ev Idx": 520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754972.375, "dur": 5.767,
    "args": {
      "External id": 2466826,"Sequence number": 24740591, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 521
    }
  },
  {
    "ph": "f", "id": 57, "pid": 1336756, "tid": 1381189, "ts": 1514656754972.375,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754972.996, "dur": 4.967,
    "args": {
      "External id": 2466827,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754973.516, "dur": 3.966,
    "args": {
      "External id": 2466828,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656754976.856, "dur": 0.517,
    "args": {
      "External id": 2466829,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755025.161, "dur": 11.239,
    "args": {
      "External id": 2466830,"Record function id": 0, "Ev Idx": 525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755027.753, "dur": 7.621,
    "args": {
      "External id": 2466831,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755030.641, "dur": 4.193,
    "args": {
      "External id": 2466832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755031.724, "dur": 2.872,
    "args": {
      "External id": 2466833,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755042.988, "dur": 9.618,
    "args": {
      "External id": 2466834,"Record function id": 0, "Sequence number": 24740590, "Fwd thread id": 1, "Ev Idx": 529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755043.901, "dur": 5.632,
    "args": {
      "External id": 2466835,"Sequence number": 24740590, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 530
    }
  },
  {
    "ph": "f", "id": 58, "pid": 1336756, "tid": 1381189, "ts": 1514656755043.901,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755045.015, "dur": 4.351,
    "args": {
      "External id": 2466836,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755047.901, "dur": 1.369,
    "args": {
      "External id": 2466837,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755055.713, "dur": 105.384,
    "args": {
      "External id": 2466838,"Record function id": 0, "Sequence number": 24740589, "Fwd thread id": 1, "Ev Idx": 533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755056.622, "dur": 96.256,
    "args": {
      "External id": 2466839,"Sequence number": 24740589, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 534
    }
  },
  {
    "ph": "f", "id": 59, "pid": 1336756, "tid": 1381189, "ts": 1514656755056.622,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755059.587, "dur": 4.455,
    "args": {
      "External id": 2466840,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755060.267, "dur": 3.372,
    "args": {
      "External id": 2466841,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755063.057, "dur": 0.462,
    "args": {
      "External id": 2466842,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755067.224, "dur": 38.266,
    "args": {
      "External id": 2466843,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755106.583, "dur": 2.435,
    "args": {
      "External id": 2466844,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755107.021, "dur": 1.502,
    "args": {
      "External id": 2466845,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755107.829, "dur": 0.574,
    "args": {
      "External id": 2466846,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755110.183, "dur": 6.786,
    "args": {
      "External id": 2466847,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755111.009, "dur": 5.445,
    "args": {
      "External id": 2466848,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755113.991, "dur": 2.372,
    "args": {
      "External id": 2466849,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755117.369, "dur": 34.724,
    "args": {
      "External id": 2466850,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755165.448, "dur": 34.552,
    "args": {
      "External id": 2466851,"Record function id": 0, "Sequence number": 24740588, "Fwd thread id": 1, "Ev Idx": 546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755166.530, "dur": 5.595,
    "args": {
      "External id": 2466852,"Sequence number": 24740588, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 547
    }
  },
  {
    "ph": "f", "id": 60, "pid": 1336756, "tid": 1381189, "ts": 1514656755166.530,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755169.991, "dur": 1.986,
    "args": {
      "External id": 2466853,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755170.656, "dur": 1.182,
    "args": {
      "External id": 2466854,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755175.252, "dur": 21.838,
    "args": {
      "External id": 2466855,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755203.601, "dur": 9.701,
    "args": {
      "External id": 2466856,"Record function id": 0, "Sequence number": 24740587, "Fwd thread id": 1, "Ev Idx": 551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755204.773, "dur": 6.480,
    "args": {
      "External id": 2466857,"Sequence number": 24740587, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 552
    }
  },
  {
    "ph": "f", "id": 61, "pid": 1336756, "tid": 1381189, "ts": 1514656755204.773,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755205.811, "dur": 5.225,
    "args": {
      "External id": 2466858,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755206.344, "dur": 4.218,
    "args": {
      "External id": 2466859,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755210.052, "dur": 0.373,
    "args": {
      "External id": 2466860,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755217.443, "dur": 7.525,
    "args": {
      "External id": 2466861,"Record function id": 0, "Ev Idx": 556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755221.062, "dur": 3.382,
    "args": {
      "External id": 2466862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755222.191, "dur": 1.627,
    "args": {
      "External id": 2466863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755222.629, "dur": 1.073,
    "args": {
      "External id": 2466864,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755228.017, "dur": 6.419,
    "args": {
      "External id": 2466865,"Record function id": 0, "Sequence number": 24740586, "Fwd thread id": 1, "Ev Idx": 560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755229.036, "dur": 2.742,
    "args": {
      "External id": 2466866,"Sequence number": 24740586, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 561
    }
  },
  {
    "ph": "f", "id": 62, "pid": 1336756, "tid": 1381189, "ts": 1514656755229.036,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755229.985, "dur": 1.664,
    "args": {
      "External id": 2466867,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755230.636, "dur": 0.881,
    "args": {
      "External id": 2466868,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755237.520, "dur": 92.567,
    "args": {
      "External id": 2466869,"Record function id": 0, "Sequence number": 24740585, "Fwd thread id": 1, "Ev Idx": 564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755240.611, "dur": 80.848,
    "args": {
      "External id": 2466870,"Sequence number": 24740585, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 565
    }
  },
  {
    "ph": "f", "id": 63, "pid": 1336756, "tid": 1381189, "ts": 1514656755240.611,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755243.304, "dur": 2.192,
    "args": {
      "External id": 2466871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755243.657, "dur": 1.423,
    "args": {
      "External id": 2466872,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755244.452, "dur": 0.504,
    "args": {
      "External id": 2466873,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755246.179, "dur": 28.037,
    "args": {
      "External id": 2466874,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755275.249, "dur": 7.930,
    "args": {
      "External id": 2466875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755275.735, "dur": 6.943,
    "args": {
      "External id": 2466876,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755282.080, "dur": 0.501,
    "args": {
      "External id": 2466877,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755284.416, "dur": 2.850,
    "args": {
      "External id": 2466878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755285.360, "dur": 1.348,
    "args": {
      "External id": 2466879,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755286.102, "dur": 0.520,
    "args": {
      "External id": 2466880,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755287.728, "dur": 33.068,
    "args": {
      "External id": 2466881,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755334.177, "dur": 28.511,
    "args": {
      "External id": 2466882,"Record function id": 0, "Sequence number": 24740584, "Fwd thread id": 1, "Ev Idx": 577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755335.247, "dur": 5.826,
    "args": {
      "External id": 2466883,"Sequence number": 24740584, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 578
    }
  },
  {
    "ph": "f", "id": 64, "pid": 1336756, "tid": 1381189, "ts": 1514656755335.247,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755338.805, "dur": 2.119,
    "args": {
      "External id": 2466884,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755339.569, "dur": 1.222,
    "args": {
      "External id": 2466885,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755343.638, "dur": 16.966,
    "args": {
      "External id": 2466886,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755366.119, "dur": 10.890,
    "args": {
      "External id": 2466887,"Record function id": 0, "Sequence number": 24740583, "Fwd thread id": 1, "Ev Idx": 582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755367.215, "dur": 7.815,
    "args": {
      "External id": 2466888,"Sequence number": 24740583, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 583
    }
  },
  {
    "ph": "f", "id": 65, "pid": 1336756, "tid": 1381189, "ts": 1514656755367.215,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755367.868, "dur": 6.929,
    "args": {
      "External id": 2466889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755370.616, "dur": 3.719,
    "args": {
      "External id": 2466890,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755373.331, "dur": 0.890,
    "args": {
      "External id": 2466891,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755380.731, "dur": 7.061,
    "args": {
      "External id": 2466892,"Record function id": 0, "Ev Idx": 587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755382.170, "dur": 5.116,
    "args": {
      "External id": 2466893,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755382.930, "dur": 3.991,
    "args": {
      "External id": 2466894,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755385.403, "dur": 1.409,
    "args": {
      "External id": 2466895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755392.125, "dur": 400.480,
    "args": {
      "External id": 2466896,"Record function id": 0, "Sequence number": 24740582, "Fwd thread id": 1, "Ev Idx": 591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755393.343, "dur": 364.208,
    "args": {
      "External id": 2466897,"Sequence number": 24740582, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 592
    }
  },
  {
    "ph": "f", "id": 66, "pid": 1336756, "tid": 1381189, "ts": 1514656755393.343,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755427.019, "dur": 1.515,
    "args": {
      "External id": 2466898,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755427.441, "dur": 0.926,
    "args": {
      "External id": 2466899,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755444.145, "dur": 5.864,
    "args": {
      "External id": 2466900,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755459.471, "dur": 1.721,
    "args": {
      "External id": 2466901,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755646.497, "dur": 2.802,
    "args": {
      "External id": 2466902,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755653.592, "dur": 40.217,
    "args": {
      "External id": 2466903,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755666.684, "dur": 0.953,
    "args": {
      "External id": 2466904,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755699.279, "dur": 32.094,
    "args": {
      "External id": 2466905,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755701.153, "dur": 29.974,
    "args": {
      "External id": 2466906,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755704.953, "dur": 5.610,
    "args": {
      "External id": 2466907,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755712.447, "dur": 18.217,
    "args": {
      "External id": 2466908,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755735.503, "dur": 2.736,
    "args": {
      "External id": 2466909,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755736.812, "dur": 1.247,
    "args": {
      "External id": 2466910,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755746.164, "dur": 4.417,
    "args": {
      "External id": 2466911,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755749.678, "dur": 0.803,
    "args": {
      "External id": 2466912,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755769.625, "dur": 16.958,
    "args": {
      "External id": 2466913,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755804.547, "dur": 7.274,
    "args": {
      "External id": 2466914,"Record function id": 0, "Ev Idx": 609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755806.182, "dur": 4.992,
    "args": {
      "External id": 2466915,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755807.992, "dur": 2.413,
    "args": {
      "External id": 2466916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755808.756, "dur": 1.528,
    "args": {
      "External id": 2466917,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755817.370, "dur": 7.243,
    "args": {
      "External id": 2466918,"Record function id": 0, "Sequence number": 24740581, "Fwd thread id": 1, "Ev Idx": 613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755818.852, "dur": 1.840,
    "args": {
      "External id": 2466919,"Sequence number": 24740581, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 614
    }
  },
  {
    "ph": "f", "id": 67, "pid": 1336756, "tid": 1381189, "ts": 1514656755818.852,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755828.443, "dur": 460.036,
    "args": {
      "External id": 2466920,"Record function id": 0, "Sequence number": 24740580, "Fwd thread id": 1, "Ev Idx": 615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755829.672, "dur": 446.241,
    "args": {
      "External id": 2466921,"Sequence number": 24740580, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 616
    }
  },
  {
    "ph": "f", "id": 68, "pid": 1336756, "tid": 1381189, "ts": 1514656755829.672,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755861.990, "dur": 8.115,
    "args": {
      "External id": 2466922,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755866.953, "dur": 2.916,
    "args": {
      "External id": 2466923,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755873.746, "dur": 6.738,
    "args": {
      "External id": 2466924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755875.182, "dur": 4.741,
    "args": {
      "External id": 2466925,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755878.845, "dur": 0.942,
    "args": {
      "External id": 2466926,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755886.379, "dur": 82.894,
    "args": {
      "External id": 2466927,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755887.307, "dur": 4.351,
    "args": {
      "External id": 2466928,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755887.757, "dur": 3.488,
    "args": {
      "External id": 2466929,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755888.538, "dur": 2.633,
    "args": {
      "External id": 2466930,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755893.038, "dur": 75.724,
    "args": {
      "External id": 2466931,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755894.574, "dur": 73.302,
    "args": {
      "External id": 2466932,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755972.913, "dur": 4.929,
    "args": {
      "External id": 2466933,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656755976.587, "dur": 1.122,
    "args": {
      "External id": 2466934,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756049.388, "dur": 6.755,
    "args": {
      "External id": 2466935,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756057.434, "dur": 2.003,
    "args": {
      "External id": 2466936,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756060.056, "dur": 1.797,
    "args": {
      "External id": 2466937,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756099.636, "dur": 2.458,
    "args": {
      "External id": 2466938,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756100.195, "dur": 1.747,
    "args": {
      "External id": 2466939,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756126.933, "dur": 127.460,
    "args": {
      "External id": 2466940,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756131.981, "dur": 4.760,
    "args": {
      "External id": 2466941,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756135.047, "dur": 0.884,
    "args": {
      "External id": 2466942,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756138.146, "dur": 10.177,
    "args": {
      "External id": 2466943,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756145.386, "dur": 2.328,
    "args": {
      "External id": 2466944,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756149.583, "dur": 3.524,
    "args": {
      "External id": 2466945,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756152.361, "dur": 0.445,
    "args": {
      "External id": 2466946,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756153.699, "dur": 4.211,
    "args": {
      "External id": 2466947,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756156.871, "dur": 0.584,
    "args": {
      "External id": 2466948,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756161.232, "dur": 2.091,
    "args": {
      "External id": 2466949,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756162.552, "dur": 0.442,
    "args": {
      "External id": 2466950,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756168.137, "dur": 5.638,
    "args": {
      "External id": 2466951,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756171.298, "dur": 2.260,
    "args": {
      "External id": 2466952,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756176.662, "dur": 1.535,
    "args": {
      "External id": 2466953,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756177.568, "dur": 0.331,
    "args": {
      "External id": 2466954,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756178.686, "dur": 5.310,
    "args": {
      "External id": 2466955,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756181.387, "dur": 2.506,
    "args": {
      "External id": 2466956,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756185.158, "dur": 54.596,
    "args": {
      "External id": 2466957,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756241.321, "dur": 4.114,
    "args": {
      "External id": 2466958,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756245.945, "dur": 4.156,
    "args": {
      "External id": 2466959,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756249.174, "dur": 0.419,
    "args": {
      "External id": 2466960,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756252.142, "dur": 1.107,
    "args": {
      "External id": 2466961,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756299.502, "dur": 8.168,
    "args": {
      "External id": 2466962,"Record function id": 0, "Ev Idx": 657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756301.240, "dur": 5.778,
    "args": {
      "External id": 2466963,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756303.023, "dur": 3.295,
    "args": {
      "External id": 2466964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756303.885, "dur": 2.275,
    "args": {
      "External id": 2466965,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756311.100, "dur": 6.482,
    "args": {
      "External id": 2466966,"Record function id": 0, "Sequence number": 24740579, "Fwd thread id": 1, "Ev Idx": 661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756311.970, "dur": 3.135,
    "args": {
      "External id": 2466967,"Sequence number": 24740579, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 662
    }
  },
  {
    "ph": "f", "id": 69, "pid": 1336756, "tid": 1381189, "ts": 1514656756311.970,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756313.457, "dur": 1.407,
    "args": {
      "External id": 2466968,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756314.059, "dur": 0.676,
    "args": {
      "External id": 2466969,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756320.810, "dur": 122.299,
    "args": {
      "External id": 2466970,"Record function id": 0, "Sequence number": 24740578, "Fwd thread id": 1, "Ev Idx": 665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756321.571, "dur": 114.317,
    "args": {
      "External id": 2466971,"Sequence number": 24740578, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 666
    }
  },
  {
    "ph": "f", "id": 70, "pid": 1336756, "tid": 1381189, "ts": 1514656756321.571,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756326.724, "dur": 5.512,
    "args": {
      "External id": 2466972,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756327.897, "dur": 3.806,
    "args": {
      "External id": 2466973,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756331.145, "dur": 0.391,
    "args": {
      "External id": 2466974,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756333.315, "dur": 42.081,
    "args": {
      "External id": 2466975,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756376.530, "dur": 5.551,
    "args": {
      "External id": 2466976,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756377.106, "dur": 4.421,
    "args": {
      "External id": 2466977,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756380.604, "dur": 0.793,
    "args": {
      "External id": 2466978,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756383.679, "dur": 3.254,
    "args": {
      "External id": 2466979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756384.586, "dur": 1.761,
    "args": {
      "External id": 2466980,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756385.640, "dur": 0.644,
    "args": {
      "External id": 2466981,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756387.532, "dur": 47.553,
    "args": {
      "External id": 2466982,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756447.952, "dur": 8.232,
    "args": {
      "External id": 2466983,"Record function id": 0, "Sequence number": 24740577, "Fwd thread id": 1, "Ev Idx": 678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756448.957, "dur": 5.830,
    "args": {
      "External id": 2466984,"Sequence number": 24740577, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 679
    }
  },
  {
    "ph": "f", "id": 71, "pid": 1336756, "tid": 1381189, "ts": 1514656756448.957,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756452.523, "dur": 2.112,
    "args": {
      "External id": 2466985,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756453.304, "dur": 1.184,
    "args": {
      "External id": 2466986,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756459.332, "dur": 9.504,
    "args": {
      "External id": 2466987,"Record function id": 0, "Sequence number": 24740576, "Fwd thread id": 1, "Ev Idx": 682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756462.085, "dur": 4.109,
    "args": {
      "External id": 2466988,"Sequence number": 24740576, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 683
    }
  },
  {
    "ph": "f", "id": 72, "pid": 1336756, "tid": 1381189, "ts": 1514656756462.085,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756463.150, "dur": 2.824,
    "args": {
      "External id": 2466989,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756463.884, "dur": 1.633,
    "args": {
      "External id": 2466990,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756464.770, "dur": 0.639,
    "args": {
      "External id": 2466991,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756472.617, "dur": 4.615,
    "args": {
      "External id": 2466992,"Record function id": 0, "Ev Idx": 687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756473.761, "dur": 3.007,
    "args": {
      "External id": 2466993,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756474.583, "dur": 1.696,
    "args": {
      "External id": 2466994,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756475.152, "dur": 0.984,
    "args": {
      "External id": 2466995,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756480.374, "dur": 10.388,
    "args": {
      "External id": 2466996,"Record function id": 0, "Sequence number": 24740575, "Fwd thread id": 1, "Ev Idx": 691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756483.677, "dur": 5.032,
    "args": {
      "External id": 2466997,"Sequence number": 24740575, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 692
    }
  },
  {
    "ph": "f", "id": 73, "pid": 1336756, "tid": 1381189, "ts": 1514656756483.677,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756484.722, "dur": 3.850,
    "args": {
      "External id": 2466998,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756487.826, "dur": 0.614,
    "args": {
      "External id": 2466999,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756493.929, "dur": 145.610,
    "args": {
      "External id": 2467000,"Record function id": 0, "Sequence number": 24740574, "Fwd thread id": 1, "Ev Idx": 695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756494.803, "dur": 133.841,
    "args": {
      "External id": 2467001,"Sequence number": 24740574, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 696
    }
  },
  {
    "ph": "f", "id": 74, "pid": 1336756, "tid": 1381189, "ts": 1514656756494.803,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756497.488, "dur": 6.007,
    "args": {
      "External id": 2467002,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756497.898, "dur": 5.156,
    "args": {
      "External id": 2467003,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756502.430, "dur": 0.497,
    "args": {
      "External id": 2467004,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756506.813, "dur": 54.060,
    "args": {
      "External id": 2467005,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756562.966, "dur": 5.502,
    "args": {
      "External id": 2467006,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756563.615, "dur": 4.157,
    "args": {
      "External id": 2467007,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756565.121, "dur": 2.494,
    "args": {
      "External id": 2467008,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756569.659, "dur": 9.004,
    "args": {
      "External id": 2467009,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756570.720, "dur": 7.417,
    "args": {
      "External id": 2467010,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756575.768, "dur": 2.281,
    "args": {
      "External id": 2467011,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756579.100, "dur": 48.880,
    "args": {
      "External id": 2467012,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756645.601, "dur": 30.564,
    "args": {
      "External id": 2467013,"Record function id": 0, "Sequence number": 24740573, "Fwd thread id": 1, "Ev Idx": 708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756646.652, "dur": 3.576,
    "args": {
      "External id": 2467014,"Sequence number": 24740573, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 709
    }
  },
  {
    "ph": "f", "id": 75, "pid": 1336756, "tid": 1381189, "ts": 1514656756646.652,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756648.130, "dur": 1.950,
    "args": {
      "External id": 2467015,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756648.707, "dur": 1.233,
    "args": {
      "External id": 2467016,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756652.980, "dur": 20.304,
    "args": {
      "External id": 2467017,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756680.035, "dur": 10.355,
    "args": {
      "External id": 2467018,"Record function id": 0, "Sequence number": 24740572, "Fwd thread id": 1, "Ev Idx": 713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756681.325, "dur": 7.307,
    "args": {
      "External id": 2467019,"Sequence number": 24740572, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 714
    }
  },
  {
    "ph": "f", "id": 76, "pid": 1336756, "tid": 1381189, "ts": 1514656756681.325,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756682.211, "dur": 6.208,
    "args": {
      "External id": 2467020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756683.144, "dur": 4.782,
    "args": {
      "External id": 2467021,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756687.232, "dur": 0.599,
    "args": {
      "External id": 2467022,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756694.295, "dur": 5.425,
    "args": {
      "External id": 2467023,"Record function id": 0, "Ev Idx": 718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756695.731, "dur": 3.506,
    "args": {
      "External id": 2467024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756696.792, "dur": 1.989,
    "args": {
      "External id": 2467025,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756697.304, "dur": 1.386,
    "args": {
      "External id": 2467026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756705.981, "dur": 452.519,
    "args": {
      "External id": 2467027,"Record function id": 0, "Sequence number": 24740571, "Fwd thread id": 1, "Ev Idx": 722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756707.219, "dur": 414.920,
    "args": {
      "External id": 2467028,"Sequence number": 24740571, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 723
    }
  },
  {
    "ph": "f", "id": 77, "pid": 1336756, "tid": 1381189, "ts": 1514656756707.219,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756728.832, "dur": 37.351,
    "args": {
      "External id": 2467029,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756730.365, "dur": 35.617,
    "args": {
      "External id": 2467030,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756732.959, "dur": 7.251,
    "args": {
      "External id": 2467031,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756736.951, "dur": 2.731,
    "args": {
      "External id": 2467032,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756741.870, "dur": 23.665,
    "args": {
      "External id": 2467033,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756777.624, "dur": 4.039,
    "args": {
      "External id": 2467034,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756780.460, "dur": 1.073,
    "args": {
      "External id": 2467035,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756785.272, "dur": 3.474,
    "args": {
      "External id": 2467036,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756786.017, "dur": 2.638,
    "args": {
      "External id": 2467037,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756799.974, "dur": 2.133,
    "args": {
      "External id": 2467038,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756812.732, "dur": 1.977,
    "args": {
      "External id": 2467039,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756971.617, "dur": 2.238,
    "args": {
      "External id": 2467040,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656756977.738, "dur": 71.638,
    "args": {
      "External id": 2467041,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757021.016, "dur": 1.171,
    "args": {
      "External id": 2467042,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757055.552, "dur": 31.994,
    "args": {
      "External id": 2467043,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757057.697, "dur": 29.669,
    "args": {
      "External id": 2467044,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757063.850, "dur": 5.213,
    "args": {
      "External id": 2467045,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757070.429, "dur": 16.457,
    "args": {
      "External id": 2467046,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757092.624, "dur": 4.724,
    "args": {
      "External id": 2467047,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757095.771, "dur": 1.431,
    "args": {
      "External id": 2467048,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757103.649, "dur": 1.995,
    "args": {
      "External id": 2467049,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757104.540, "dur": 0.993,
    "args": {
      "External id": 2467050,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757109.769, "dur": 3.176,
    "args": {
      "External id": 2467051,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757110.209, "dur": 2.655,
    "args": {
      "External id": 2467052,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757137.941, "dur": 19.164,
    "args": {
      "External id": 2467053,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757171.403, "dur": 11.165,
    "args": {
      "External id": 2467054,"Record function id": 0, "Ev Idx": 749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757173.393, "dur": 8.555,
    "args": {
      "External id": 2467055,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757175.398, "dur": 5.793,
    "args": {
      "External id": 2467056,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757179.223, "dur": 1.878,
    "args": {
      "External id": 2467057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757186.112, "dur": 7.635,
    "args": {
      "External id": 2467058,"Record function id": 0, "Sequence number": 24740570, "Fwd thread id": 1, "Ev Idx": 753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757187.153, "dur": 3.668,
    "args": {
      "External id": 2467059,"Sequence number": 24740570, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 754
    }
  },
  {
    "ph": "f", "id": 78, "pid": 1336756, "tid": 1381189, "ts": 1514656757187.153,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757188.828, "dur": 1.813,
    "args": {
      "External id": 2467060,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757189.424, "dur": 1.065,
    "args": {
      "External id": 2467061,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757197.125, "dur": 128.359,
    "args": {
      "External id": 2467062,"Record function id": 0, "Sequence number": 24740569, "Fwd thread id": 1, "Ev Idx": 757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757197.879, "dur": 120.422,
    "args": {
      "External id": 2467063,"Sequence number": 24740569, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 758
    }
  },
  {
    "ph": "f", "id": 79, "pid": 1336756, "tid": 1381189, "ts": 1514656757197.879,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757201.959, "dur": 8.516,
    "args": {
      "External id": 2467064,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757205.964, "dur": 3.903,
    "args": {
      "External id": 2467065,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757208.967, "dur": 0.706,
    "args": {
      "External id": 2467066,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757211.767, "dur": 53.907,
    "args": {
      "External id": 2467067,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757266.860, "dur": 5.196,
    "args": {
      "External id": 2467068,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757267.517, "dur": 3.981,
    "args": {
      "External id": 2467069,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757268.331, "dur": 2.970,
    "args": {
      "External id": 2467070,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757275.796, "dur": 3.095,
    "args": {
      "External id": 2467071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757276.971, "dur": 1.265,
    "args": {
      "External id": 2467072,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757277.810, "dur": 0.350,
    "args": {
      "External id": 2467073,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757279.376, "dur": 38.206,
    "args": {
      "External id": 2467074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757330.181, "dur": 11.991,
    "args": {
      "External id": 2467075,"Record function id": 0, "Sequence number": 24740568, "Fwd thread id": 1, "Ev Idx": 770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757331.134, "dur": 8.173,
    "args": {
      "External id": 2467076,"Sequence number": 24740568, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 771
    }
  },
  {
    "ph": "f", "id": 80, "pid": 1336756, "tid": 1381189, "ts": 1514656757331.134,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757335.078, "dur": 4.074,
    "args": {
      "External id": 2467077,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757335.599, "dur": 3.415,
    "args": {
      "External id": 2467078,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757347.237, "dur": 8.147,
    "args": {
      "External id": 2467079,"Record function id": 0, "Sequence number": 24740567, "Fwd thread id": 1, "Ev Idx": 774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757349.808, "dur": 3.369,
    "args": {
      "External id": 2467080,"Sequence number": 24740567, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 775
    }
  },
  {
    "ph": "f", "id": 81, "pid": 1336756, "tid": 1381189, "ts": 1514656757349.808,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757350.648, "dur": 2.340,
    "args": {
      "External id": 2467081,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757351.316, "dur": 1.184,
    "args": {
      "External id": 2467082,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757352.000, "dur": 0.384,
    "args": {
      "External id": 2467083,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757359.206, "dur": 4.713,
    "args": {
      "External id": 2467084,"Record function id": 0, "Ev Idx": 779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757360.413, "dur": 3.031,
    "args": {
      "External id": 2467085,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757361.309, "dur": 1.875,
    "args": {
      "External id": 2467086,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757362.004, "dur": 1.089,
    "args": {
      "External id": 2467087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757366.955, "dur": 10.764,
    "args": {
      "External id": 2467088,"Record function id": 0, "Sequence number": 24740566, "Fwd thread id": 1, "Ev Idx": 783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757367.779, "dur": 7.394,
    "args": {
      "External id": 2467089,"Sequence number": 24740566, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 784
    }
  },
  {
    "ph": "f", "id": 82, "pid": 1336756, "tid": 1381189, "ts": 1514656757367.779,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757370.891, "dur": 4.141,
    "args": {
      "External id": 2467090,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757373.726, "dur": 1.174,
    "args": {
      "External id": 2467091,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757381.818, "dur": 313.404,
    "args": {
      "External id": 2467092,"Record function id": 0, "Sequence number": 24740565, "Fwd thread id": 1, "Ev Idx": 787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757382.824, "dur": 288.568,
    "args": {
      "External id": 2467093,"Sequence number": 24740565, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 788
    }
  },
  {
    "ph": "f", "id": 83, "pid": 1336756, "tid": 1381189, "ts": 1514656757382.824,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757398.622, "dur": 5.353,
    "args": {
      "External id": 2467094,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757400.442, "dur": 3.054,
    "args": {
      "External id": 2467095,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757405.952, "dur": 4.616,
    "args": {
      "External id": 2467096,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757408.612, "dur": 1.750,
    "args": {
      "External id": 2467097,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757411.931, "dur": 4.608,
    "args": {
      "External id": 2467098,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757412.688, "dur": 3.689,
    "args": {
      "External id": 2467099,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757443.178, "dur": 201.866,
    "args": {
      "External id": 2467100,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757523.257, "dur": 2.977,
    "args": {
      "External id": 2467101,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757547.310, "dur": 5.281,
    "args": {
      "External id": 2467102,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757657.963, "dur": 3.956,
    "args": {
      "External id": 2467103,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757665.066, "dur": 0.659,
    "args": {
      "External id": 2467104,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757667.829, "dur": 0.704,
    "args": {
      "External id": 2467105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757705.509, "dur": 228.981,
    "args": {
      "External id": 2467106,"Record function id": 0, "Sequence number": 24740564, "Fwd thread id": 1, "Ev Idx": 801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757707.246, "dur": 221.030,
    "args": {
      "External id": 2467107,"Sequence number": 24740564, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 802
    }
  },
  {
    "ph": "f", "id": 84, "pid": 1336756, "tid": 1381189, "ts": 1514656757707.246,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757729.985, "dur": 46.394,
    "args": {
      "External id": 2467108,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757732.464, "dur": 2.944,
    "args": {
      "External id": 2467109,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757737.035, "dur": 38.758,
    "args": {
      "External id": 2467110,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757786.171, "dur": 5.737,
    "args": {
      "External id": 2467111,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757789.662, "dur": 1.865,
    "args": {
      "External id": 2467112,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757941.607, "dur": 202.613,
    "args": {
      "External id": 2467113,"Record function id": 0, "Sequence number": 24740563, "Fwd thread id": 1, "Ev Idx": 808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757942.984, "dur": 194.104,
    "args": {
      "External id": 2467114,"Sequence number": 24740563, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 809
    }
  },
  {
    "ph": "f", "id": 85, "pid": 1336756, "tid": 1381189, "ts": 1514656757942.984,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757955.732, "dur": 68.376,
    "args": {
      "External id": 2467115,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757957.744, "dur": 2.136,
    "args": {
      "External id": 2467116,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656757960.850, "dur": 62.171,
    "args": {
      "External id": 2467117,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758033.526, "dur": 9.433,
    "args": {
      "External id": 2467118,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758039.530, "dur": 3.068,
    "args": {
      "External id": 2467119,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758152.840, "dur": 16.020,
    "args": {
      "External id": 2467120,"Record function id": 0, "Sequence number": 24740562, "Fwd thread id": 1, "Ev Idx": 815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758154.555, "dur": 10.838,
    "args": {
      "External id": 2467121,"Sequence number": 24740562, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 816
    }
  },
  {
    "ph": "f", "id": 86, "pid": 1336756, "tid": 1381189, "ts": 1514656758154.555,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758156.626, "dur": 8.484,
    "args": {
      "External id": 2467122,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758158.160, "dur": 6.787,
    "args": {
      "External id": 2467123,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758172.494, "dur": 11.255,
    "args": {
      "External id": 2467124,"Record function id": 0, "Sequence number": 24740561, "Fwd thread id": 1, "Ev Idx": 819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758176.581, "dur": 4.731,
    "args": {
      "External id": 2467125,"Sequence number": 24740561, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 820
    }
  },
  {
    "ph": "f", "id": 87, "pid": 1336756, "tid": 1381189, "ts": 1514656758176.581,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758177.588, "dur": 3.564,
    "args": {
      "External id": 2467126,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758180.130, "dur": 0.894,
    "args": {
      "External id": 2467127,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758188.885, "dur": 5.953,
    "args": {
      "External id": 2467128,"Record function id": 0, "Sequence number": 24740560, "Fwd thread id": 1, "Ev Idx": 823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758189.827, "dur": 2.388,
    "args": {
      "External id": 2467129,"Sequence number": 24740560, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 824
    }
  },
  {
    "ph": "f", "id": 88, "pid": 1336756, "tid": 1381189, "ts": 1514656758189.827,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758190.745, "dur": 1.334,
    "args": {
      "External id": 2467130,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758191.219, "dur": 0.733,
    "args": {
      "External id": 2467131,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758198.105, "dur": 10.196,
    "args": {
      "External id": 2467132,"Record function id": 0, "Sequence number": 24740559, "Fwd thread id": 1, "Ev Idx": 827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758198.985, "dur": 6.706,
    "args": {
      "External id": 2467133,"Sequence number": 24740559, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 828
    }
  },
  {
    "ph": "f", "id": 89, "pid": 1336756, "tid": 1381189, "ts": 1514656758198.985,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758199.947, "dur": 5.580,
    "args": {
      "External id": 2467134,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758204.386, "dur": 1.052,
    "args": {
      "External id": 2467135,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758211.756, "dur": 156.295,
    "args": {
      "External id": 2467136,"Record function id": 0, "Sequence number": 24740558, "Fwd thread id": 1, "Ev Idx": 831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758212.837, "dur": 147.130,
    "args": {
      "External id": 2467137,"Sequence number": 24740558, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 832
    }
  },
  {
    "ph": "f", "id": 90, "pid": 1336756, "tid": 1381189, "ts": 1514656758212.837,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758217.544, "dur": 6.746,
    "args": {
      "External id": 2467138,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758219.409, "dur": 4.183,
    "args": {
      "External id": 2467139,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758221.452, "dur": 1.820,
    "args": {
      "External id": 2467140,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758225.856, "dur": 70.863,
    "args": {
      "External id": 2467141,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758298.142, "dur": 6.386,
    "args": {
      "External id": 2467142,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758301.180, "dur": 2.616,
    "args": {
      "External id": 2467143,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758302.500, "dur": 1.093,
    "args": {
      "External id": 2467144,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758306.308, "dur": 5.647,
    "args": {
      "External id": 2467145,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758307.324, "dur": 3.848,
    "args": {
      "External id": 2467146,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758310.598, "dur": 0.460,
    "args": {
      "External id": 2467147,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758312.586, "dur": 46.496,
    "args": {
      "External id": 2467148,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758373.122, "dur": 8.421,
    "args": {
      "External id": 2467149,"Record function id": 0, "Sequence number": 24740557, "Fwd thread id": 1, "Ev Idx": 844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758374.076, "dur": 5.658,
    "args": {
      "External id": 2467150,"Sequence number": 24740557, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 845
    }
  },
  {
    "ph": "f", "id": 91, "pid": 1336756, "tid": 1381189, "ts": 1514656758374.076,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758377.864, "dur": 1.707,
    "args": {
      "External id": 2467151,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758378.451, "dur": 1.012,
    "args": {
      "External id": 2467152,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758385.064, "dur": 7.243,
    "args": {
      "External id": 2467153,"Record function id": 0, "Sequence number": 24740556, "Fwd thread id": 1, "Ev Idx": 848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758385.937, "dur": 3.640,
    "args": {
      "External id": 2467154,"Sequence number": 24740556, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 849
    }
  },
  {
    "ph": "f", "id": 92, "pid": 1336756, "tid": 1381189, "ts": 1514656758385.937,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758386.682, "dur": 2.680,
    "args": {
      "External id": 2467155,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758387.190, "dur": 1.722,
    "args": {
      "External id": 2467156,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758388.200, "dur": 0.597,
    "args": {
      "External id": 2467157,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758400.975, "dur": 11.438,
    "args": {
      "External id": 2467158,"Record function id": 0, "Ev Idx": 853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758402.592, "dur": 8.971,
    "args": {
      "External id": 2467159,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758404.907, "dur": 6.241,
    "args": {
      "External id": 2467160,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758408.236, "dur": 2.812,
    "args": {
      "External id": 2467161,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758415.630, "dur": 5.604,
    "args": {
      "External id": 2467162,"Record function id": 0, "Sequence number": 24740555, "Fwd thread id": 1, "Ev Idx": 857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758416.503, "dur": 2.729,
    "args": {
      "External id": 2467163,"Sequence number": 24740555, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 858
    }
  },
  {
    "ph": "f", "id": 93, "pid": 1336756, "tid": 1381189, "ts": 1514656758416.503,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758417.544, "dur": 1.538,
    "args": {
      "External id": 2467164,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758418.164, "dur": 0.786,
    "args": {
      "External id": 2467165,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758424.207, "dur": 93.132,
    "args": {
      "External id": 2467166,"Record function id": 0, "Sequence number": 24740554, "Fwd thread id": 1, "Ev Idx": 861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758425.299, "dur": 85.626,
    "args": {
      "External id": 2467167,"Sequence number": 24740554, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 862
    }
  },
  {
    "ph": "f", "id": 94, "pid": 1336756, "tid": 1381189, "ts": 1514656758425.299,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758427.248, "dur": 4.487,
    "args": {
      "External id": 2467168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758430.015, "dur": 1.256,
    "args": {
      "External id": 2467169,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758430.695, "dur": 0.465,
    "args": {
      "External id": 2467170,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758432.566, "dur": 27.964,
    "args": {
      "External id": 2467171,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758461.566, "dur": 5.582,
    "args": {
      "External id": 2467172,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758462.128, "dur": 4.543,
    "args": {
      "External id": 2467173,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758465.485, "dur": 1.067,
    "args": {
      "External id": 2467174,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758470.483, "dur": 5.549,
    "args": {
      "External id": 2467175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758472.410, "dur": 3.048,
    "args": {
      "External id": 2467176,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758473.078, "dur": 2.314,
    "args": {
      "External id": 2467177,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758476.692, "dur": 33.479,
    "args": {
      "External id": 2467178,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758521.609, "dur": 57.480,
    "args": {
      "External id": 2467179,"Record function id": 0, "Sequence number": 24740553, "Fwd thread id": 1, "Ev Idx": 874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758522.609, "dur": 22.958,
    "args": {
      "External id": 2467180,"Sequence number": 24740553, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 875
    }
  },
  {
    "ph": "f", "id": 95, "pid": 1336756, "tid": 1381189, "ts": 1514656758522.609,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758524.171, "dur": 21.202,
    "args": {
      "External id": 2467181,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758526.619, "dur": 18.295,
    "args": {
      "External id": 2467182,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758550.171, "dur": 24.693,
    "args": {
      "External id": 2467183,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758584.490, "dur": 7.742,
    "args": {
      "External id": 2467184,"Record function id": 0, "Sequence number": 24740552, "Fwd thread id": 1, "Ev Idx": 879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758585.756, "dur": 4.035,
    "args": {
      "External id": 2467185,"Sequence number": 24740552, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 880
    }
  },
  {
    "ph": "f", "id": 96, "pid": 1336756, "tid": 1381189, "ts": 1514656758585.756,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758586.604, "dur": 2.977,
    "args": {
      "External id": 2467186,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758587.210, "dur": 1.789,
    "args": {
      "External id": 2467187,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758588.461, "dur": 0.434,
    "args": {
      "External id": 2467188,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758596.147, "dur": 7.545,
    "args": {
      "External id": 2467189,"Record function id": 0, "Ev Idx": 884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758597.379, "dur": 5.744,
    "args": {
      "External id": 2467190,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758598.543, "dur": 4.185,
    "args": {
      "External id": 2467191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758601.309, "dur": 1.289,
    "args": {
      "External id": 2467192,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758606.718, "dur": 8.695,
    "args": {
      "External id": 2467193,"Record function id": 0, "Sequence number": 24740551, "Fwd thread id": 1, "Ev Idx": 888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758607.625, "dur": 5.560,
    "args": {
      "External id": 2467194,"Sequence number": 24740551, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 889
    }
  },
  {
    "ph": "f", "id": 97, "pid": 1336756, "tid": 1381189, "ts": 1514656758607.625,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758608.559, "dur": 4.484,
    "args": {
      "External id": 2467195,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758611.978, "dur": 0.925,
    "args": {
      "External id": 2467196,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758620.707, "dur": 127.328,
    "args": {
      "External id": 2467197,"Record function id": 0, "Sequence number": 24740550, "Fwd thread id": 1, "Ev Idx": 892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758621.644, "dur": 116.841,
    "args": {
      "External id": 2467198,"Sequence number": 24740550, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 893
    }
  },
  {
    "ph": "f", "id": 98, "pid": 1336756, "tid": 1381189, "ts": 1514656758621.644,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758624.336, "dur": 4.513,
    "args": {
      "External id": 2467199,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758624.951, "dur": 3.496,
    "args": {
      "External id": 2467200,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758627.875, "dur": 0.458,
    "args": {
      "External id": 2467201,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758629.517, "dur": 32.027,
    "args": {
      "External id": 2467202,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758662.692, "dur": 2.761,
    "args": {
      "External id": 2467203,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758663.129, "dur": 1.829,
    "args": {
      "External id": 2467204,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758663.804, "dur": 1.038,
    "args": {
      "External id": 2467205,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758666.678, "dur": 10.369,
    "args": {
      "External id": 2467206,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758673.622, "dur": 3.073,
    "args": {
      "External id": 2467207,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758676.242, "dur": 0.337,
    "args": {
      "External id": 2467208,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758677.498, "dur": 60.293,
    "args": {
      "External id": 2467209,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758753.182, "dur": 28.432,
    "args": {
      "External id": 2467210,"Record function id": 0, "Sequence number": 24740549, "Fwd thread id": 1, "Ev Idx": 905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758754.103, "dur": 3.635,
    "args": {
      "External id": 2467211,"Sequence number": 24740549, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 906
    }
  },
  {
    "ph": "f", "id": 99, "pid": 1336756, "tid": 1381189, "ts": 1514656758754.103,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758755.629, "dur": 1.970,
    "args": {
      "External id": 2467212,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758756.350, "dur": 1.134,
    "args": {
      "External id": 2467213,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758759.934, "dur": 19.392,
    "args": {
      "External id": 2467214,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758785.073, "dur": 10.556,
    "args": {
      "External id": 2467215,"Record function id": 0, "Sequence number": 24740548, "Fwd thread id": 1, "Ev Idx": 910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758788.023, "dur": 5.440,
    "args": {
      "External id": 2467216,"Sequence number": 24740548, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 911
    }
  },
  {
    "ph": "f", "id": 100, "pid": 1336756, "tid": 1381189, "ts": 1514656758788.023,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758788.823, "dur": 4.460,
    "args": {
      "External id": 2467217,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758789.689, "dur": 3.134,
    "args": {
      "External id": 2467218,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758792.184, "dur": 0.543,
    "args": {
      "External id": 2467219,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758799.586, "dur": 5.557,
    "args": {
      "External id": 2467220,"Record function id": 0, "Ev Idx": 915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758801.009, "dur": 3.600,
    "args": {
      "External id": 2467221,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758801.997, "dur": 2.265,
    "args": {
      "External id": 2467222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758803.049, "dur": 1.106,
    "args": {
      "External id": 2467223,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758808.991, "dur": 405.595,
    "args": {
      "External id": 2467224,"Record function id": 0, "Sequence number": 24740547, "Fwd thread id": 1, "Ev Idx": 919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758810.169, "dur": 369.262,
    "args": {
      "External id": 2467225,"Sequence number": 24740547, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 920
    }
  },
  {
    "ph": "f", "id": 101, "pid": 1336756, "tid": 1381189, "ts": 1514656758810.169,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758845.348, "dur": 3.805,
    "args": {
      "External id": 2467226,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758847.882, "dur": 1.091,
    "args": {
      "External id": 2467227,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758863.167, "dur": 5.245,
    "args": {
      "External id": 2467228,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656758877.480, "dur": 1.930,
    "args": {
      "External id": 2467229,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759065.279, "dur": 3.092,
    "args": {
      "External id": 2467230,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759072.498, "dur": 39.930,
    "args": {
      "External id": 2467231,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759084.548, "dur": 0.987,
    "args": {
      "External id": 2467232,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759121.868, "dur": 36.158,
    "args": {
      "External id": 2467233,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759123.704, "dur": 34.126,
    "args": {
      "External id": 2467234,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759132.423, "dur": 5.410,
    "args": {
      "External id": 2467235,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759139.477, "dur": 17.757,
    "args": {
      "External id": 2467236,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759162.504, "dur": 2.138,
    "args": {
      "External id": 2467237,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759163.484, "dur": 1.055,
    "args": {
      "External id": 2467238,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759170.775, "dur": 1.615,
    "args": {
      "External id": 2467239,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759171.503, "dur": 0.753,
    "args": {
      "External id": 2467240,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759193.580, "dur": 16.635,
    "args": {
      "External id": 2467241,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759227.131, "dur": 11.436,
    "args": {
      "External id": 2467242,"Record function id": 0, "Ev Idx": 937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759231.524, "dur": 6.103,
    "args": {
      "External id": 2467243,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759233.317, "dur": 3.182,
    "args": {
      "External id": 2467244,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759234.620, "dur": 1.772,
    "args": {
      "External id": 2467245,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759242.265, "dur": 6.647,
    "args": {
      "External id": 2467246,"Record function id": 0, "Sequence number": 24740546, "Fwd thread id": 1, "Ev Idx": 941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759243.829, "dur": 1.453,
    "args": {
      "External id": 2467247,"Sequence number": 24740546, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 942
    }
  },
  {
    "ph": "f", "id": 102, "pid": 1336756, "tid": 1381189, "ts": 1514656759243.829,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759252.761, "dur": 467.063,
    "args": {
      "External id": 2467248,"Record function id": 0, "Sequence number": 24740545, "Fwd thread id": 1, "Ev Idx": 943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759254.002, "dur": 450.861,
    "args": {
      "External id": 2467249,"Sequence number": 24740545, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 944
    }
  },
  {
    "ph": "f", "id": 103, "pid": 1336756, "tid": 1381189, "ts": 1514656759254.002,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759286.423, "dur": 7.690,
    "args": {
      "External id": 2467250,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759290.987, "dur": 2.836,
    "args": {
      "External id": 2467251,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759297.358, "dur": 6.809,
    "args": {
      "External id": 2467252,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759298.829, "dur": 4.618,
    "args": {
      "External id": 2467253,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759302.490, "dur": 0.788,
    "args": {
      "External id": 2467254,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759307.816, "dur": 104.490,
    "args": {
      "External id": 2467255,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759308.777, "dur": 6.433,
    "args": {
      "External id": 2467256,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759311.444, "dur": 3.312,
    "args": {
      "External id": 2467257,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759312.131, "dur": 2.513,
    "args": {
      "External id": 2467258,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759316.428, "dur": 95.491,
    "args": {
      "External id": 2467259,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759317.887, "dur": 93.135,
    "args": {
      "External id": 2467260,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759415.821, "dur": 2.067,
    "args": {
      "External id": 2467261,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759416.545, "dur": 1.181,
    "args": {
      "External id": 2467262,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759450.545, "dur": 4.756,
    "args": {
      "External id": 2467263,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759458.197, "dur": 1.947,
    "args": {
      "External id": 2467264,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759460.843, "dur": 1.983,
    "args": {
      "External id": 2467265,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759495.640, "dur": 1.556,
    "args": {
      "External id": 2467266,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759496.185, "dur": 0.839,
    "args": {
      "External id": 2467267,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759519.864, "dur": 166.101,
    "args": {
      "External id": 2467268,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759524.817, "dur": 25.450,
    "args": {
      "External id": 2467269,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759547.637, "dur": 1.311,
    "args": {
      "External id": 2467270,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759551.604, "dur": 10.199,
    "args": {
      "External id": 2467271,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759558.309, "dur": 2.734,
    "args": {
      "External id": 2467272,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759563.249, "dur": 1.515,
    "args": {
      "External id": 2467273,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759564.098, "dur": 0.340,
    "args": {
      "External id": 2467274,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759567.550, "dur": 1.885,
    "args": {
      "External id": 2467275,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759568.401, "dur": 0.510,
    "args": {
      "External id": 2467276,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759574.758, "dur": 1.750,
    "args": {
      "External id": 2467277,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759575.725, "dur": 0.459,
    "args": {
      "External id": 2467278,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759577.287, "dur": 6.985,
    "args": {
      "External id": 2467279,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759582.077, "dur": 2.013,
    "args": {
      "External id": 2467280,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759584.858, "dur": 4.814,
    "args": {
      "External id": 2467281,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759587.823, "dur": 1.569,
    "args": {
      "External id": 2467282,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759591.977, "dur": 3.164,
    "args": {
      "External id": 2467283,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759592.570, "dur": 2.483,
    "args": {
      "External id": 2467284,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759596.357, "dur": 71.992,
    "args": {
      "External id": 2467285,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759672.694, "dur": 3.109,
    "args": {
      "External id": 2467286,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759676.670, "dur": 5.115,
    "args": {
      "External id": 2467287,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759680.460, "dur": 0.426,
    "args": {
      "External id": 2467288,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759684.089, "dur": 0.896,
    "args": {
      "External id": 2467289,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759730.930, "dur": 9.500,
    "args": {
      "External id": 2467290,"Record function id": 0, "Ev Idx": 985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759733.129, "dur": 6.490,
    "args": {
      "External id": 2467291,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759734.849, "dur": 3.714,
    "args": {
      "External id": 2467292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759736.323, "dur": 2.081,
    "args": {
      "External id": 2467293,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759744.039, "dur": 8.494,
    "args": {
      "External id": 2467294,"Record function id": 0, "Sequence number": 24740544, "Fwd thread id": 1, "Ev Idx": 989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759744.904, "dur": 4.905,
    "args": {
      "External id": 2467295,"Sequence number": 24740544, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 990
    }
  },
  {
    "ph": "f", "id": 104, "pid": 1336756, "tid": 1381189, "ts": 1514656759744.904,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759748.073, "dur": 1.568,
    "args": {
      "External id": 2467296,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759748.697, "dur": 0.810,
    "args": {
      "External id": 2467297,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759755.758, "dur": 129.258,
    "args": {
      "External id": 2467298,"Record function id": 0, "Sequence number": 24740543, "Fwd thread id": 1, "Ev Idx": 993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759756.570, "dur": 121.006,
    "args": {
      "External id": 2467299,"Sequence number": 24740543, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 994
    }
  },
  {
    "ph": "f", "id": 105, "pid": 1336756, "tid": 1381189, "ts": 1514656759756.570,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759760.479, "dur": 5.569,
    "args": {
      "External id": 2467300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759761.661, "dur": 3.850,
    "args": {
      "External id": 2467301,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759764.728, "dur": 0.615,
    "args": {
      "External id": 2467302,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759767.320, "dur": 44.719,
    "args": {
      "External id": 2467303,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759813.370, "dur": 5.218,
    "args": {
      "External id": 2467304,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759813.911, "dur": 4.181,
    "args": {
      "External id": 2467305,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759816.966, "dur": 0.973,
    "args": {
      "External id": 2467306,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759819.985, "dur": 5.696,
    "args": {
      "External id": 2467307,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759821.141, "dur": 3.923,
    "args": {
      "External id": 2467308,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759824.329, "dur": 0.657,
    "args": {
      "External id": 2467309,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759826.395, "dur": 50.422,
    "args": {
      "External id": 2467310,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759889.843, "dur": 8.339,
    "args": {
      "External id": 2467311,"Record function id": 0, "Sequence number": 24740542, "Fwd thread id": 1, "Ev Idx": 1006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759890.971, "dur": 5.572,
    "args": {
      "External id": 2467312,"Sequence number": 24740542, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1007
    }
  },
  {
    "ph": "f", "id": 106, "pid": 1336756, "tid": 1381189, "ts": 1514656759890.971,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759892.089, "dur": 4.303,
    "args": {
      "External id": 2467313,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759894.660, "dur": 1.602,
    "args": {
      "External id": 2467314,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759901.361, "dur": 9.532,
    "args": {
      "External id": 2467315,"Record function id": 0, "Sequence number": 24740541, "Fwd thread id": 1, "Ev Idx": 1010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759902.282, "dur": 6.491,
    "args": {
      "External id": 2467316,"Sequence number": 24740541, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1011
    }
  },
  {
    "ph": "f", "id": 107, "pid": 1336756, "tid": 1381189, "ts": 1514656759902.282,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759903.179, "dur": 5.391,
    "args": {
      "External id": 2467317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759903.937, "dur": 4.160,
    "args": {
      "External id": 2467318,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759907.456, "dur": 0.534,
    "args": {
      "External id": 2467319,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759914.499, "dur": 5.042,
    "args": {
      "External id": 2467320,"Record function id": 0, "Ev Idx": 1015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759915.763, "dur": 3.241,
    "args": {
      "External id": 2467321,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759916.759, "dur": 1.775,
    "args": {
      "External id": 2467322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759917.476, "dur": 0.963,
    "args": {
      "External id": 2467323,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759924.598, "dur": 7.562,
    "args": {
      "External id": 2467324,"Record function id": 0, "Sequence number": 24740540, "Fwd thread id": 1, "Ev Idx": 1019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759925.634, "dur": 4.440,
    "args": {
      "External id": 2467325,"Sequence number": 24740540, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1020
    }
  },
  {
    "ph": "f", "id": 108, "pid": 1336756, "tid": 1381189, "ts": 1514656759925.634,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759926.637, "dur": 3.296,
    "args": {
      "External id": 2467326,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759929.200, "dur": 0.600,
    "args": {
      "External id": 2467327,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759935.441, "dur": 189.132,
    "args": {
      "External id": 2467328,"Record function id": 0, "Sequence number": 24740539, "Fwd thread id": 1, "Ev Idx": 1023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759961.298, "dur": 152.422,
    "args": {
      "External id": 2467329,"Sequence number": 24740539, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1024
    }
  },
  {
    "ph": "f", "id": 109, "pid": 1336756, "tid": 1381189, "ts": 1514656759961.298,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759965.961, "dur": 2.082,
    "args": {
      "External id": 2467330,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759966.349, "dur": 1.285,
    "args": {
      "External id": 2467331,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759967.205, "dur": 0.296,
    "args": {
      "External id": 2467332,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656759968.723, "dur": 74.038,
    "args": {
      "External id": 2467333,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760045.101, "dur": 7.890,
    "args": {
      "External id": 2467334,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760045.830, "dur": 6.279,
    "args": {
      "External id": 2467335,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760049.671, "dur": 2.252,
    "args": {
      "External id": 2467336,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760054.654, "dur": 6.855,
    "args": {
      "External id": 2467337,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760056.027, "dur": 4.871,
    "args": {
      "External id": 2467338,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760058.498, "dur": 2.320,
    "args": {
      "External id": 2467339,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760061.956, "dur": 50.990,
    "args": {
      "External id": 2467340,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760130.770, "dur": 35.298,
    "args": {
      "External id": 2467341,"Record function id": 0, "Sequence number": 24740538, "Fwd thread id": 1, "Ev Idx": 1036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760131.827, "dur": 5.644,
    "args": {
      "External id": 2467342,"Sequence number": 24740538, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1037
    }
  },
  {
    "ph": "f", "id": 110, "pid": 1336756, "tid": 1381189, "ts": 1514656760131.827,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760135.253, "dur": 2.077,
    "args": {
      "External id": 2467343,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760136.063, "dur": 1.134,
    "args": {
      "External id": 2467344,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760140.602, "dur": 22.277,
    "args": {
      "External id": 2467345,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760169.475, "dur": 8.271,
    "args": {
      "External id": 2467346,"Record function id": 0, "Sequence number": 24740537, "Fwd thread id": 1, "Ev Idx": 1041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760170.408, "dur": 5.337,
    "args": {
      "External id": 2467347,"Sequence number": 24740537, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1042
    }
  },
  {
    "ph": "f", "id": 111, "pid": 1336756, "tid": 1381189, "ts": 1514656760170.408,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760171.004, "dur": 4.550,
    "args": {
      "External id": 2467348,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760171.606, "dur": 3.424,
    "args": {
      "External id": 2467349,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760174.300, "dur": 0.636,
    "args": {
      "External id": 2467350,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760181.741, "dur": 7.964,
    "args": {
      "External id": 2467351,"Record function id": 0, "Ev Idx": 1046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760182.982, "dur": 6.233,
    "args": {
      "External id": 2467352,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760184.098, "dur": 4.619,
    "args": {
      "External id": 2467353,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760187.101, "dur": 1.484,
    "args": {
      "External id": 2467354,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760193.802, "dur": 432.010,
    "args": {
      "External id": 2467355,"Record function id": 0, "Sequence number": 24740536, "Fwd thread id": 1, "Ev Idx": 1050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760195.153, "dur": 393.359,
    "args": {
      "External id": 2467356,"Sequence number": 24740536, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1051
    }
  },
  {
    "ph": "f", "id": 112, "pid": 1336756, "tid": 1381189, "ts": 1514656760195.153,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760219.775, "dur": 33.217,
    "args": {
      "External id": 2467357,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760221.085, "dur": 31.673,
    "args": {
      "External id": 2467358,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760223.717, "dur": 5.074,
    "args": {
      "External id": 2467359,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760225.830, "dur": 2.495,
    "args": {
      "External id": 2467360,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760230.066, "dur": 22.231,
    "args": {
      "External id": 2467361,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760266.619, "dur": 1.928,
    "args": {
      "External id": 2467362,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760267.336, "dur": 1.050,
    "args": {
      "External id": 2467363,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760272.129, "dur": 5.241,
    "args": {
      "External id": 2467364,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760274.860, "dur": 2.412,
    "args": {
      "External id": 2467365,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760288.283, "dur": 2.107,
    "args": {
      "External id": 2467366,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760301.541, "dur": 2.350,
    "args": {
      "External id": 2467367,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760452.925, "dur": 2.014,
    "args": {
      "External id": 2467368,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760461.101, "dur": 31.191,
    "args": {
      "External id": 2467369,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760469.522, "dur": 0.658,
    "args": {
      "External id": 2467370,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760497.459, "dur": 52.403,
    "args": {
      "External id": 2467371,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760501.551, "dur": 48.066,
    "args": {
      "External id": 2467372,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760507.743, "dur": 4.623,
    "args": {
      "External id": 2467373,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760513.595, "dur": 34.969,
    "args": {
      "External id": 2467374,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760556.260, "dur": 3.098,
    "args": {
      "External id": 2467375,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760557.575, "dur": 1.673,
    "args": {
      "External id": 2467376,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760566.395, "dur": 4.240,
    "args": {
      "External id": 2467377,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760569.162, "dur": 1.355,
    "args": {
      "External id": 2467378,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760572.730, "dur": 6.526,
    "args": {
      "External id": 2467379,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760575.903, "dur": 3.260,
    "args": {
      "External id": 2467380,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760605.341, "dur": 19.071,
    "args": {
      "External id": 2467381,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760637.897, "dur": 9.241,
    "args": {
      "External id": 2467382,"Record function id": 0, "Ev Idx": 1077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760639.971, "dur": 6.455,
    "args": {
      "External id": 2467383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760641.968, "dur": 3.590,
    "args": {
      "External id": 2467384,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760643.163, "dur": 2.294,
    "args": {
      "External id": 2467385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760650.767, "dur": 9.436,
    "args": {
      "External id": 2467386,"Record function id": 0, "Sequence number": 24740535, "Fwd thread id": 1, "Ev Idx": 1081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760651.777, "dur": 5.442,
    "args": {
      "External id": 2467387,"Sequence number": 24740535, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1082
    }
  },
  {
    "ph": "f", "id": 113, "pid": 1336756, "tid": 1381189, "ts": 1514656760651.777,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760655.146, "dur": 1.895,
    "args": {
      "External id": 2467388,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760655.918, "dur": 0.959,
    "args": {
      "External id": 2467389,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760663.579, "dur": 131.183,
    "args": {
      "External id": 2467390,"Record function id": 0, "Sequence number": 24740534, "Fwd thread id": 1, "Ev Idx": 1085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760664.616, "dur": 123.169,
    "args": {
      "External id": 2467391,"Sequence number": 24740534, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1086
    }
  },
  {
    "ph": "f", "id": 114, "pid": 1336756, "tid": 1381189, "ts": 1514656760664.616,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760668.713, "dur": 6.341,
    "args": {
      "External id": 2467392,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760670.248, "dur": 4.249,
    "args": {
      "External id": 2467393,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760673.484, "dur": 0.869,
    "args": {
      "External id": 2467394,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760676.267, "dur": 54.460,
    "args": {
      "External id": 2467395,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760731.981, "dur": 7.546,
    "args": {
      "External id": 2467396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760732.663, "dur": 6.282,
    "args": {
      "External id": 2467397,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760736.000, "dur": 2.769,
    "args": {
      "External id": 2467398,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760740.962, "dur": 5.871,
    "args": {
      "External id": 2467399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760742.252, "dur": 4.068,
    "args": {
      "External id": 2467400,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760745.583, "dur": 0.675,
    "args": {
      "External id": 2467401,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760747.320, "dur": 39.731,
    "args": {
      "External id": 2467402,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760799.896, "dur": 10.003,
    "args": {
      "External id": 2467403,"Record function id": 0, "Sequence number": 24740533, "Fwd thread id": 1, "Ev Idx": 1098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760800.953, "dur": 6.646,
    "args": {
      "External id": 2467404,"Sequence number": 24740533, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1099
    }
  },
  {
    "ph": "f", "id": 115, "pid": 1336756, "tid": 1381189, "ts": 1514656760800.953,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760802.276, "dur": 5.169,
    "args": {
      "External id": 2467405,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760804.320, "dur": 3.024,
    "args": {
      "External id": 2467406,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760815.504, "dur": 9.509,
    "args": {
      "External id": 2467407,"Record function id": 0, "Sequence number": 24740532, "Fwd thread id": 1, "Ev Idx": 1102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760816.545, "dur": 6.808,
    "args": {
      "External id": 2467408,"Sequence number": 24740532, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1103
    }
  },
  {
    "ph": "f", "id": 116, "pid": 1336756, "tid": 1381189, "ts": 1514656760816.545,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760817.644, "dur": 5.500,
    "args": {
      "External id": 2467409,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760818.240, "dur": 4.395,
    "args": {
      "External id": 2467410,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760821.713, "dur": 0.763,
    "args": {
      "External id": 2467411,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760828.823, "dur": 4.992,
    "args": {
      "External id": 2467412,"Record function id": 0, "Ev Idx": 1107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760830.420, "dur": 2.876,
    "args": {
      "External id": 2467413,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760831.324, "dur": 1.481,
    "args": {
      "External id": 2467414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760831.770, "dur": 0.942,
    "args": {
      "External id": 2467415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760838.499, "dur": 6.311,
    "args": {
      "External id": 2467416,"Record function id": 0, "Sequence number": 24740531, "Fwd thread id": 1, "Ev Idx": 1111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760839.328, "dur": 3.244,
    "args": {
      "External id": 2467417,"Sequence number": 24740531, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1112
    }
  },
  {
    "ph": "f", "id": 117, "pid": 1336756, "tid": 1381189, "ts": 1514656760839.328,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760840.535, "dur": 1.894,
    "args": {
      "External id": 2467418,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760840.947, "dur": 1.348,
    "args": {
      "External id": 2467419,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760851.426, "dur": 345.838,
    "args": {
      "External id": 2467420,"Record function id": 0, "Sequence number": 24740530, "Fwd thread id": 1, "Ev Idx": 1115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760852.476, "dur": 320.016,
    "args": {
      "External id": 2467421,"Sequence number": 24740530, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1116
    }
  },
  {
    "ph": "f", "id": 118, "pid": 1336756, "tid": 1381189, "ts": 1514656760852.476,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760867.251, "dur": 5.910,
    "args": {
      "External id": 2467422,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760869.224, "dur": 3.437,
    "args": {
      "External id": 2467423,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760875.459, "dur": 6.441,
    "args": {
      "External id": 2467424,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760878.143, "dur": 3.490,
    "args": {
      "External id": 2467425,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760892.313, "dur": 4.953,
    "args": {
      "External id": 2467426,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760895.301, "dur": 1.705,
    "args": {
      "External id": 2467427,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656760925.646, "dur": 220.350,
    "args": {
      "External id": 2467428,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761041.951, "dur": 6.349,
    "args": {
      "External id": 2467429,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761050.295, "dur": 4.284,
    "args": {
      "External id": 2467430,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761159.206, "dur": 3.867,
    "args": {
      "External id": 2467431,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761166.242, "dur": 0.800,
    "args": {
      "External id": 2467432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761169.046, "dur": 0.942,
    "args": {
      "External id": 2467433,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761207.577, "dur": 229.610,
    "args": {
      "External id": 2467434,"Record function id": 0, "Sequence number": 24740529, "Fwd thread id": 1, "Ev Idx": 1129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761209.075, "dur": 220.695,
    "args": {
      "External id": 2467435,"Sequence number": 24740529, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1130
    }
  },
  {
    "ph": "f", "id": 119, "pid": 1336756, "tid": 1381189, "ts": 1514656761209.075,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761228.408, "dur": 45.553,
    "args": {
      "External id": 2467436,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761231.313, "dur": 3.193,
    "args": {
      "External id": 2467437,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761236.049, "dur": 37.388,
    "args": {
      "External id": 2467438,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761283.193, "dur": 6.247,
    "args": {
      "External id": 2467439,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761286.470, "dur": 2.641,
    "args": {
      "External id": 2467440,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761444.460, "dur": 184.648,
    "args": {
      "External id": 2467441,"Record function id": 0, "Sequence number": 24740528, "Fwd thread id": 1, "Ev Idx": 1136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761445.906, "dur": 175.925,
    "args": {
      "External id": 2467442,"Sequence number": 24740528, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1137
    }
  },
  {
    "ph": "f", "id": 120, "pid": 1336756, "tid": 1381189, "ts": 1514656761445.906,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761457.368, "dur": 43.999,
    "args": {
      "External id": 2467443,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761459.366, "dur": 2.623,
    "args": {
      "External id": 2467444,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761468.167, "dur": 32.693,
    "args": {
      "External id": 2467445,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761508.336, "dur": 5.607,
    "args": {
      "External id": 2467446,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761511.596, "dur": 2.015,
    "args": {
      "External id": 2467447,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761637.762, "dur": 19.874,
    "args": {
      "External id": 2467448,"Record function id": 0, "Sequence number": 24740527, "Fwd thread id": 1, "Ev Idx": 1143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761639.810, "dur": 14.359,
    "args": {
      "External id": 2467449,"Sequence number": 24740527, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1144
    }
  },
  {
    "ph": "f", "id": 121, "pid": 1336756, "tid": 1381189, "ts": 1514656761639.810,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761642.366, "dur": 11.495,
    "args": {
      "External id": 2467450,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761646.181, "dur": 7.484,
    "args": {
      "External id": 2467451,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761661.351, "dur": 8.510,
    "args": {
      "External id": 2467452,"Record function id": 0, "Sequence number": 24740526, "Fwd thread id": 1, "Ev Idx": 1147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761662.312, "dur": 5.372,
    "args": {
      "External id": 2467453,"Sequence number": 24740526, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1148
    }
  },
  {
    "ph": "f", "id": 122, "pid": 1336756, "tid": 1381189, "ts": 1514656761662.312,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761663.522, "dur": 4.019,
    "args": {
      "External id": 2467454,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761666.486, "dur": 0.858,
    "args": {
      "External id": 2467455,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761673.468, "dur": 6.320,
    "args": {
      "External id": 2467456,"Record function id": 0, "Sequence number": 24740525, "Fwd thread id": 1, "Ev Idx": 1151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761674.295, "dur": 3.180,
    "args": {
      "External id": 2467457,"Sequence number": 24740525, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1152
    }
  },
  {
    "ph": "f", "id": 123, "pid": 1336756, "tid": 1381189, "ts": 1514656761674.295,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761675.756, "dur": 1.571,
    "args": {
      "External id": 2467458,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761676.619, "dur": 0.617,
    "args": {
      "External id": 2467459,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761683.069, "dur": 9.823,
    "args": {
      "External id": 2467460,"Record function id": 0, "Sequence number": 24740524, "Fwd thread id": 1, "Ev Idx": 1155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761684.127, "dur": 6.285,
    "args": {
      "External id": 2467461,"Sequence number": 24740524, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1156
    }
  },
  {
    "ph": "f", "id": 124, "pid": 1336756, "tid": 1381189, "ts": 1514656761684.127,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761685.234, "dur": 5.034,
    "args": {
      "External id": 2467462,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761688.883, "dur": 1.292,
    "args": {
      "External id": 2467463,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761696.514, "dur": 163.124,
    "args": {
      "External id": 2467464,"Record function id": 0, "Sequence number": 24740523, "Fwd thread id": 1, "Ev Idx": 1159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761698.170, "dur": 153.796,
    "args": {
      "External id": 2467465,"Sequence number": 24740523, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1160
    }
  },
  {
    "ph": "f", "id": 125, "pid": 1336756, "tid": 1381189, "ts": 1514656761698.170,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761702.248, "dur": 6.757,
    "args": {
      "External id": 2467466,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761704.171, "dur": 4.142,
    "args": {
      "External id": 2467467,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761706.400, "dur": 1.603,
    "args": {
      "External id": 2467468,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761712.543, "dur": 72.195,
    "args": {
      "External id": 2467469,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761785.943, "dur": 4.559,
    "args": {
      "External id": 2467470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761786.971, "dur": 2.867,
    "args": {
      "External id": 2467471,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761788.587, "dur": 1.086,
    "args": {
      "External id": 2467472,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761792.671, "dur": 6.297,
    "args": {
      "External id": 2467473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761794.396, "dur": 3.885,
    "args": {
      "External id": 2467474,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761797.760, "dur": 0.422,
    "args": {
      "External id": 2467475,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761801.516, "dur": 49.701,
    "args": {
      "External id": 2467476,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761866.968, "dur": 7.605,
    "args": {
      "External id": 2467477,"Record function id": 0, "Sequence number": 24740522, "Fwd thread id": 1, "Ev Idx": 1172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761867.857, "dur": 4.023,
    "args": {
      "External id": 2467478,"Sequence number": 24740522, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1173
    }
  },
  {
    "ph": "f", "id": 126, "pid": 1336756, "tid": 1381189, "ts": 1514656761867.857,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761869.346, "dur": 2.385,
    "args": {
      "External id": 2467479,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761870.137, "dur": 1.452,
    "args": {
      "External id": 2467480,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761878.029, "dur": 9.216,
    "args": {
      "External id": 2467481,"Record function id": 0, "Sequence number": 24740521, "Fwd thread id": 1, "Ev Idx": 1176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761879.056, "dur": 6.001,
    "args": {
      "External id": 2467482,"Sequence number": 24740521, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1177
    }
  },
  {
    "ph": "f", "id": 127, "pid": 1336756, "tid": 1381189, "ts": 1514656761879.056,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761880.033, "dur": 4.820,
    "args": {
      "External id": 2467483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761880.842, "dur": 3.544,
    "args": {
      "External id": 2467484,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761883.818, "dur": 0.459,
    "args": {
      "External id": 2467485,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761895.582, "dur": 9.381,
    "args": {
      "External id": 2467486,"Record function id": 0, "Ev Idx": 1181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761897.160, "dur": 7.059,
    "args": {
      "External id": 2467487,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761899.750, "dur": 4.021,
    "args": {
      "External id": 2467488,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761900.988, "dur": 2.685,
    "args": {
      "External id": 2467489,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761908.051, "dur": 6.132,
    "args": {
      "External id": 2467490,"Record function id": 0, "Sequence number": 24740520, "Fwd thread id": 1, "Ev Idx": 1185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761909.164, "dur": 2.952,
    "args": {
      "External id": 2467491,"Sequence number": 24740520, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1186
    }
  },
  {
    "ph": "f", "id": 128, "pid": 1336756, "tid": 1381189, "ts": 1514656761909.164,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761910.311, "dur": 1.672,
    "args": {
      "External id": 2467492,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761911.090, "dur": 0.787,
    "args": {
      "External id": 2467493,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761917.299, "dur": 139.735,
    "args": {
      "External id": 2467494,"Record function id": 0, "Sequence number": 24740519, "Fwd thread id": 1, "Ev Idx": 1189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761917.984, "dur": 130.869,
    "args": {
      "External id": 2467495,"Sequence number": 24740519, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1190
    }
  },
  {
    "ph": "f", "id": 129, "pid": 1336756, "tid": 1381189, "ts": 1514656761917.984,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761922.664, "dur": 2.248,
    "args": {
      "External id": 2467496,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761923.302, "dur": 1.216,
    "args": {
      "External id": 2467497,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761923.898, "dur": 0.499,
    "args": {
      "External id": 2467498,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761925.574, "dur": 26.561,
    "args": {
      "External id": 2467499,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761953.041, "dur": 6.996,
    "args": {
      "External id": 2467500,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761953.880, "dur": 5.639,
    "args": {
      "External id": 2467501,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761958.830, "dur": 0.576,
    "args": {
      "External id": 2467502,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761961.069, "dur": 4.434,
    "args": {
      "External id": 2467503,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761962.163, "dur": 2.859,
    "args": {
      "External id": 2467504,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761962.740, "dur": 2.168,
    "args": {
      "External id": 2467505,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656761966.337, "dur": 81.312,
    "args": {
      "External id": 2467506,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762063.925, "dur": 38.817,
    "args": {
      "External id": 2467507,"Record function id": 0, "Sequence number": 24740518, "Fwd thread id": 1, "Ev Idx": 1202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762064.989, "dur": 8.463,
    "args": {
      "External id": 2467508,"Sequence number": 24740518, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1203
    }
  },
  {
    "ph": "f", "id": 130, "pid": 1336756, "tid": 1381189, "ts": 1514656762064.989,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762069.004, "dur": 4.301,
    "args": {
      "External id": 2467509,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762071.464, "dur": 1.749,
    "args": {
      "External id": 2467510,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762076.571, "dur": 23.838,
    "args": {
      "External id": 2467511,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762108.643, "dur": 8.075,
    "args": {
      "External id": 2467512,"Record function id": 0, "Sequence number": 24740517, "Fwd thread id": 1, "Ev Idx": 1207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762109.626, "dur": 4.901,
    "args": {
      "External id": 2467513,"Sequence number": 24740517, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1208
    }
  },
  {
    "ph": "f", "id": 131, "pid": 1336756, "tid": 1381189, "ts": 1514656762109.626,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762110.932, "dur": 3.390,
    "args": {
      "External id": 2467514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762111.636, "dur": 2.183,
    "args": {
      "External id": 2467515,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762113.088, "dur": 0.554,
    "args": {
      "External id": 2467516,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762120.965, "dur": 7.746,
    "args": {
      "External id": 2467517,"Record function id": 0, "Ev Idx": 1212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762122.138, "dur": 6.032,
    "args": {
      "External id": 2467518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762123.366, "dur": 4.481,
    "args": {
      "External id": 2467519,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762126.077, "dur": 1.691,
    "args": {
      "External id": 2467520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762131.709, "dur": 12.082,
    "args": {
      "External id": 2467521,"Record function id": 0, "Sequence number": 24740516, "Fwd thread id": 1, "Ev Idx": 1216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762132.958, "dur": 8.586,
    "args": {
      "External id": 2467522,"Sequence number": 24740516, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1217
    }
  },
  {
    "ph": "f", "id": 132, "pid": 1336756, "tid": 1381189, "ts": 1514656762132.958,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762134.524, "dur": 6.882,
    "args": {
      "External id": 2467523,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762140.233, "dur": 1.029,
    "args": {
      "External id": 2467524,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762146.951, "dur": 102.214,
    "args": {
      "External id": 2467525,"Record function id": 0, "Sequence number": 24740515, "Fwd thread id": 1, "Ev Idx": 1220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762147.760, "dur": 91.754,
    "args": {
      "External id": 2467526,"Sequence number": 24740515, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1221
    }
  },
  {
    "ph": "f", "id": 133, "pid": 1336756, "tid": 1381189, "ts": 1514656762147.760,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762152.944, "dur": 2.740,
    "args": {
      "External id": 2467527,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762153.368, "dur": 1.931,
    "args": {
      "External id": 2467528,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762154.432, "dur": 0.766,
    "args": {
      "External id": 2467529,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762156.461, "dur": 33.968,
    "args": {
      "External id": 2467530,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762191.500, "dur": 7.473,
    "args": {
      "External id": 2467531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762192.501, "dur": 5.919,
    "args": {
      "External id": 2467532,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762195.743, "dur": 2.512,
    "args": {
      "External id": 2467533,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762200.195, "dur": 4.768,
    "args": {
      "External id": 2467534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762201.441, "dur": 2.880,
    "args": {
      "External id": 2467535,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762203.809, "dur": 0.446,
    "args": {
      "External id": 2467536,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762205.437, "dur": 33.343,
    "args": {
      "External id": 2467537,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762253.782, "dur": 25.020,
    "args": {
      "External id": 2467538,"Record function id": 0, "Sequence number": 24740514, "Fwd thread id": 1, "Ev Idx": 1233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762254.759, "dur": 3.381,
    "args": {
      "External id": 2467539,"Sequence number": 24740514, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1234
    }
  },
  {
    "ph": "f", "id": 134, "pid": 1336756, "tid": 1381189, "ts": 1514656762254.759,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762255.985, "dur": 2.019,
    "args": {
      "External id": 2467540,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762256.964, "dur": 0.907,
    "args": {
      "External id": 2467541,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762260.732, "dur": 16.017,
    "args": {
      "External id": 2467542,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762284.717, "dur": 6.867,
    "args": {
      "External id": 2467543,"Record function id": 0, "Sequence number": 24740513, "Fwd thread id": 1, "Ev Idx": 1238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762285.528, "dur": 3.824,
    "args": {
      "External id": 2467544,"Sequence number": 24740513, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1239
    }
  },
  {
    "ph": "f", "id": 135, "pid": 1336756, "tid": 1381189, "ts": 1514656762285.528,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762286.536, "dur": 2.616,
    "args": {
      "External id": 2467545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762287.164, "dur": 1.526,
    "args": {
      "External id": 2467546,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762288.015, "dur": 0.557,
    "args": {
      "External id": 2467547,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762297.304, "dur": 4.609,
    "args": {
      "External id": 2467548,"Record function id": 0, "Ev Idx": 1243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762298.401, "dur": 3.048,
    "args": {
      "External id": 2467549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762299.195, "dur": 1.986,
    "args": {
      "External id": 2467550,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762299.670, "dur": 1.427,
    "args": {
      "External id": 2467551,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762307.957, "dur": 377.423,
    "args": {
      "External id": 2467552,"Record function id": 0, "Sequence number": 24740512, "Fwd thread id": 1, "Ev Idx": 1247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762309.425, "dur": 344.448,
    "args": {
      "External id": 2467553,"Sequence number": 24740512, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1248
    }
  },
  {
    "ph": "f", "id": 136, "pid": 1336756, "tid": 1381189, "ts": 1514656762309.425,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762345.239, "dur": 1.787,
    "args": {
      "External id": 2467554,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762345.775, "dur": 1.104,
    "args": {
      "External id": 2467555,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762361.041, "dur": 6.280,
    "args": {
      "External id": 2467556,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762376.254, "dur": 1.897,
    "args": {
      "External id": 2467557,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762524.156, "dur": 1.750,
    "args": {
      "External id": 2467558,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762548.506, "dur": 38.050,
    "args": {
      "External id": 2467559,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762559.202, "dur": 1.030,
    "args": {
      "External id": 2467560,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762592.496, "dur": 36.774,
    "args": {
      "External id": 2467561,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762596.560, "dur": 32.476,
    "args": {
      "External id": 2467562,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762602.419, "dur": 5.733,
    "args": {
      "External id": 2467563,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762609.667, "dur": 18.759,
    "args": {
      "External id": 2467564,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762633.919, "dur": 2.577,
    "args": {
      "External id": 2467565,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762635.028, "dur": 1.344,
    "args": {
      "External id": 2467566,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762642.917, "dur": 3.933,
    "args": {
      "External id": 2467567,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762645.621, "dur": 1.141,
    "args": {
      "External id": 2467568,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762664.986, "dur": 16.423,
    "args": {
      "External id": 2467569,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762696.804, "dur": 8.446,
    "args": {
      "External id": 2467570,"Record function id": 0, "Ev Idx": 1265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762698.884, "dur": 5.769,
    "args": {
      "External id": 2467571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762700.661, "dur": 2.987,
    "args": {
      "External id": 2467572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762701.556, "dur": 1.998,
    "args": {
      "External id": 2467573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762708.887, "dur": 6.454,
    "args": {
      "External id": 2467574,"Record function id": 0, "Sequence number": 24740511, "Fwd thread id": 1, "Ev Idx": 1269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762710.450, "dur": 1.660,
    "args": {
      "External id": 2467575,"Sequence number": 24740511, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1270
    }
  },
  {
    "ph": "f", "id": 137, "pid": 1336756, "tid": 1381189, "ts": 1514656762710.450,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762719.363, "dur": 470.394,
    "args": {
      "External id": 2467576,"Record function id": 0, "Sequence number": 24740510, "Fwd thread id": 1, "Ev Idx": 1271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762723.023, "dur": 452.239,
    "args": {
      "External id": 2467577,"Sequence number": 24740510, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1272
    }
  },
  {
    "ph": "f", "id": 138, "pid": 1336756, "tid": 1381189, "ts": 1514656762723.023,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762756.044, "dur": 7.804,
    "args": {
      "External id": 2467578,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762760.870, "dur": 2.753,
    "args": {
      "External id": 2467579,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 1274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762767.147, "dur": 6.364,
    "args": {
      "External id": 2467580,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762768.807, "dur": 4.131,
    "args": {
      "External id": 2467581,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762769.930, "dur": 2.824,
    "args": {
      "External id": 2467582,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762777.105, "dur": 88.812,
    "args": {
      "External id": 2467583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 1278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762780.183, "dur": 5.477,
    "args": {
      "External id": 2467584,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 1279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762780.852, "dur": 4.395,
    "args": {
      "External id": 2467585,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762784.465, "dur": 0.709,
    "args": {
      "External id": 2467586,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762787.004, "dur": 78.420,
    "args": {
      "External id": 2467587,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762788.770, "dur": 75.685,
    "args": {
      "External id": 2467588,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762869.742, "dur": 2.528,
    "args": {
      "External id": 2467589,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 1284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762870.743, "dur": 1.415,
    "args": {
      "External id": 2467590,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762903.567, "dur": 5.181,
    "args": {
      "External id": 2467591,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762911.542, "dur": 1.736,
    "args": {
      "External id": 2467592,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762914.112, "dur": 1.850,
    "args": {
      "External id": 2467593,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762948.878, "dur": 2.246,
    "args": {
      "External id": 2467594,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762949.760, "dur": 1.214,
    "args": {
      "External id": 2467595,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762973.046, "dur": 180.031,
    "args": {
      "External id": 2467596,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 1291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762978.052, "dur": 51.184,
    "args": {
      "External id": 2467597,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656762983.157, "dur": 44.383,
    "args": {
      "External id": 2467598,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763031.548, "dur": 6.594,
    "args": {
      "External id": 2467599,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 1294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763036.531, "dur": 0.700,
    "args": {
      "External id": 2467600,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 1295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763039.768, "dur": 2.184,
    "args": {
      "External id": 2467601,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763041.045, "dur": 0.572,
    "args": {
      "External id": 2467602,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763045.259, "dur": 2.347,
    "args": {
      "External id": 2467603,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763046.461, "dur": 0.679,
    "args": {
      "External id": 2467604,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 1299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763051.236, "dur": 4.090,
    "args": {
      "External id": 2467605,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 1300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763054.367, "dur": 0.656,
    "args": {
      "External id": 2467606,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 1301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763056.299, "dur": 7.794,
    "args": {
      "External id": 2467607,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 1302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763061.711, "dur": 2.170,
    "args": {
      "External id": 2467608,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 1303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763066.779, "dur": 3.982,
    "args": {
      "External id": 2467609,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 1304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763070.133, "dur": 0.353,
    "args": {
      "External id": 2467610,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 1305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763071.446, "dur": 3.995,
    "args": {
      "External id": 2467611,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763072.246, "dur": 3.085,
    "args": {
      "External id": 2467612,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763076.693, "dur": 61.984,
    "args": {
      "External id": 2467613,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 1308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763142.489, "dur": 1.158,
    "args": {
      "External id": 2467614,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 1309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763144.409, "dur": 2.402,
    "args": {
      "External id": 2467615,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 1310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763145.709, "dur": 0.639,
    "args": {
      "External id": 2467616,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 1311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763151.349, "dur": 0.733,
    "args": {
      "External id": 2467617,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 1312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763201.756, "dur": 9.424,
    "args": {
      "External id": 2467618,"Record function id": 0, "Ev Idx": 1313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763203.896, "dur": 6.513,
    "args": {
      "External id": 2467619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763205.793, "dur": 3.750,
    "args": {
      "External id": 2467620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763206.927, "dur": 2.518,
    "args": {
      "External id": 2467621,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763214.910, "dur": 7.079,
    "args": {
      "External id": 2467622,"Record function id": 0, "Sequence number": 24740509, "Fwd thread id": 1, "Ev Idx": 1317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763215.995, "dur": 3.378,
    "args": {
      "External id": 2467623,"Sequence number": 24740509, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1318
    }
  },
  {
    "ph": "f", "id": 139, "pid": 1336756, "tid": 1381189, "ts": 1514656763215.995,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763217.534, "dur": 1.660,
    "args": {
      "External id": 2467624,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763218.233, "dur": 0.867,
    "args": {
      "External id": 2467625,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763225.525, "dur": 123.968,
    "args": {
      "External id": 2467626,"Record function id": 0, "Sequence number": 24740508, "Fwd thread id": 1, "Ev Idx": 1321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763226.515, "dur": 115.768,
    "args": {
      "External id": 2467627,"Sequence number": 24740508, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1322
    }
  },
  {
    "ph": "f", "id": 140, "pid": 1336756, "tid": 1381189, "ts": 1514656763226.515,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763230.599, "dur": 5.629,
    "args": {
      "External id": 2467628,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763231.938, "dur": 3.765,
    "args": {
      "External id": 2467629,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763234.978, "dur": 0.574,
    "args": {
      "External id": 2467630,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763237.358, "dur": 42.074,
    "args": {
      "External id": 2467631,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763280.602, "dur": 5.603,
    "args": {
      "External id": 2467632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763281.503, "dur": 4.053,
    "args": {
      "External id": 2467633,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763284.356, "dur": 1.049,
    "args": {
      "External id": 2467634,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763287.746, "dur": 6.456,
    "args": {
      "External id": 2467635,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763290.568, "dur": 3.041,
    "args": {
      "External id": 2467636,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763291.195, "dur": 2.330,
    "args": {
      "External id": 2467637,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763294.670, "dur": 46.903,
    "args": {
      "External id": 2467638,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763354.502, "dur": 6.657,
    "args": {
      "External id": 2467639,"Record function id": 0, "Sequence number": 24740507, "Fwd thread id": 1, "Ev Idx": 1334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763355.745, "dur": 3.684,
    "args": {
      "External id": 2467640,"Sequence number": 24740507, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1335
    }
  },
  {
    "ph": "f", "id": 141, "pid": 1336756, "tid": 1381189, "ts": 1514656763355.745,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763357.099, "dur": 2.173,
    "args": {
      "External id": 2467641,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763358.001, "dur": 1.166,
    "args": {
      "External id": 2467642,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763364.776, "dur": 12.371,
    "args": {
      "External id": 2467643,"Record function id": 0, "Sequence number": 24740506, "Fwd thread id": 1, "Ev Idx": 1338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763365.765, "dur": 8.649,
    "args": {
      "External id": 2467644,"Sequence number": 24740506, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1339
    }
  },
  {
    "ph": "f", "id": 142, "pid": 1336756, "tid": 1381189, "ts": 1514656763365.765,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763368.600, "dur": 5.615,
    "args": {
      "External id": 2467645,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763369.221, "dur": 4.509,
    "args": {
      "External id": 2467646,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763373.264, "dur": 0.337,
    "args": {
      "External id": 2467647,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763380.870, "dur": 5.400,
    "args": {
      "External id": 2467648,"Record function id": 0, "Ev Idx": 1343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763382.024, "dur": 3.744,
    "args": {
      "External id": 2467649,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763383.164, "dur": 2.273,
    "args": {
      "External id": 2467650,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763383.791, "dur": 1.443,
    "args": {
      "External id": 2467651,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763389.304, "dur": 8.241,
    "args": {
      "External id": 2467652,"Record function id": 0, "Sequence number": 24740505, "Fwd thread id": 1, "Ev Idx": 1347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763390.072, "dur": 4.868,
    "args": {
      "External id": 2467653,"Sequence number": 24740505, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1348
    }
  },
  {
    "ph": "f", "id": 143, "pid": 1336756, "tid": 1381189, "ts": 1514656763390.072,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763391.464, "dur": 3.339,
    "args": {
      "External id": 2467654,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763394.043, "dur": 0.632,
    "args": {
      "External id": 2467655,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763400.643, "dur": 111.835,
    "args": {
      "External id": 2467656,"Record function id": 0, "Sequence number": 24740504, "Fwd thread id": 1, "Ev Idx": 1351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763401.494, "dur": 101.748,
    "args": {
      "External id": 2467657,"Sequence number": 24740504, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1352
    }
  },
  {
    "ph": "f", "id": 144, "pid": 1336756, "tid": 1381189, "ts": 1514656763401.494,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763403.999, "dur": 4.439,
    "args": {
      "External id": 2467658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763404.496, "dur": 3.477,
    "args": {
      "External id": 2467659,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763407.337, "dur": 0.532,
    "args": {
      "External id": 2467660,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763409.460, "dur": 33.131,
    "args": {
      "External id": 2467661,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763445.299, "dur": 4.999,
    "args": {
      "External id": 2467662,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763445.801, "dur": 4.010,
    "args": {
      "External id": 2467663,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763446.863, "dur": 2.762,
    "args": {
      "External id": 2467664,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763451.457, "dur": 3.181,
    "args": {
      "External id": 2467665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763452.807, "dur": 1.371,
    "args": {
      "External id": 2467666,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763453.798, "dur": 0.307,
    "args": {
      "External id": 2467667,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763457.279, "dur": 45.273,
    "args": {
      "External id": 2467668,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763519.160, "dur": 52.968,
    "args": {
      "External id": 2467669,"Record function id": 0, "Sequence number": 24740503, "Fwd thread id": 1, "Ev Idx": 1364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763520.232, "dur": 5.065,
    "args": {
      "External id": 2467670,"Sequence number": 24740503, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1365
    }
  },
  {
    "ph": "f", "id": 145, "pid": 1336756, "tid": 1381189, "ts": 1514656763520.232,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763523.553, "dur": 1.594,
    "args": {
      "External id": 2467671,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763524.031, "dur": 0.975,
    "args": {
      "External id": 2467672,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763544.040, "dur": 24.639,
    "args": {
      "External id": 2467673,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763577.503, "dur": 11.932,
    "args": {
      "External id": 2467674,"Record function id": 0, "Sequence number": 24740502, "Fwd thread id": 1, "Ev Idx": 1369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763579.050, "dur": 7.736,
    "args": {
      "External id": 2467675,"Sequence number": 24740502, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1370
    }
  },
  {
    "ph": "f", "id": 146, "pid": 1336756, "tid": 1381189, "ts": 1514656763579.050,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763580.541, "dur": 6.016,
    "args": {
      "External id": 2467676,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763582.129, "dur": 3.818,
    "args": {
      "External id": 2467677,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763585.373, "dur": 0.425,
    "args": {
      "External id": 2467678,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763593.606, "dur": 8.160,
    "args": {
      "External id": 2467679,"Record function id": 0, "Ev Idx": 1374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763597.708, "dur": 3.592,
    "args": {
      "External id": 2467680,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763599.033, "dur": 1.945,
    "args": {
      "External id": 2467681,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763599.557, "dur": 1.348,
    "args": {
      "External id": 2467682,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763606.038, "dur": 466.067,
    "args": {
      "External id": 2467683,"Record function id": 0, "Sequence number": 24740501, "Fwd thread id": 1, "Ev Idx": 1378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763611.833, "dur": 415.135,
    "args": {
      "External id": 2467684,"Sequence number": 24740501, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1379
    }
  },
  {
    "ph": "f", "id": 147, "pid": 1336756, "tid": 1381189, "ts": 1514656763611.833,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763634.640, "dur": 33.319,
    "args": {
      "External id": 2467685,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763636.000, "dur": 31.778,
    "args": {
      "External id": 2467686,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763638.851, "dur": 5.434,
    "args": {
      "External id": 2467687,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763641.142, "dur": 2.671,
    "args": {
      "External id": 2467688,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763645.687, "dur": 21.621,
    "args": {
      "External id": 2467689,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763682.066, "dur": 7.050,
    "args": {
      "External id": 2467690,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763684.982, "dur": 4.008,
    "args": {
      "External id": 2467691,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763693.115, "dur": 1.828,
    "args": {
      "External id": 2467692,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763693.928, "dur": 0.911,
    "args": {
      "External id": 2467693,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763707.705, "dur": 2.218,
    "args": {
      "External id": 2467694,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763720.315, "dur": 2.784,
    "args": {
      "External id": 2467695,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763876.419, "dur": 3.507,
    "args": {
      "External id": 2467696,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763884.158, "dur": 33.249,
    "args": {
      "External id": 2467697,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763893.959, "dur": 0.812,
    "args": {
      "External id": 2467698,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763922.916, "dur": 30.305,
    "args": {
      "External id": 2467699,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763924.805, "dur": 28.211,
    "args": {
      "External id": 2467700,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763928.625, "dur": 3.688,
    "args": {
      "External id": 2467701,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763935.981, "dur": 16.509,
    "args": {
      "External id": 2467702,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763957.710, "dur": 4.174,
    "args": {
      "External id": 2467703,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763960.740, "dur": 1.009,
    "args": {
      "External id": 2467704,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763967.992, "dur": 4.004,
    "args": {
      "External id": 2467705,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763968.933, "dur": 2.953,
    "args": {
      "External id": 2467706,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763973.995, "dur": 4.140,
    "args": {
      "External id": 2467707,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656763976.736, "dur": 1.306,
    "args": {
      "External id": 2467708,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764048.564, "dur": 21.941,
    "args": {
      "External id": 2467709,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764086.516, "dur": 9.798,
    "args": {
      "External id": 2467710,"Record function id": 0, "Ev Idx": 1405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764088.589, "dur": 6.951,
    "args": {
      "External id": 2467711,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764090.687, "dur": 3.736,
    "args": {
      "External id": 2467712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764092.035, "dur": 2.290,
    "args": {
      "External id": 2467713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764100.437, "dur": 10.054,
    "args": {
      "External id": 2467714,"Record function id": 0, "Sequence number": 24740500, "Fwd thread id": 1, "Ev Idx": 1409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764101.515, "dur": 6.048,
    "args": {
      "External id": 2467715,"Sequence number": 24740500, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1410
    }
  },
  {
    "ph": "f", "id": 148, "pid": 1336756, "tid": 1381189, "ts": 1514656764101.515,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764103.068, "dur": 4.325,
    "args": {
      "External id": 2467716,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764105.829, "dur": 1.471,
    "args": {
      "External id": 2467717,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764113.988, "dur": 138.034,
    "args": {
      "External id": 2467718,"Record function id": 0, "Sequence number": 24740499, "Fwd thread id": 1, "Ev Idx": 1413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764114.821, "dur": 130.672,
    "args": {
      "External id": 2467719,"Sequence number": 24740499, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1414
    }
  },
  {
    "ph": "f", "id": 149, "pid": 1336756, "tid": 1381189, "ts": 1514656764114.821,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764118.441, "dur": 5.084,
    "args": {
      "External id": 2467720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764120.048, "dur": 2.863,
    "args": {
      "External id": 2467721,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764121.655, "dur": 1.071,
    "args": {
      "External id": 2467722,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764124.834, "dur": 60.478,
    "args": {
      "External id": 2467723,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764188.248, "dur": 7.850,
    "args": {
      "External id": 2467724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764189.194, "dur": 6.377,
    "args": {
      "External id": 2467725,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764193.144, "dur": 2.268,
    "args": {
      "External id": 2467726,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764197.594, "dur": 3.537,
    "args": {
      "External id": 2467727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764198.726, "dur": 1.709,
    "args": {
      "External id": 2467728,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764199.877, "dur": 0.463,
    "args": {
      "External id": 2467729,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764203.811, "dur": 40.929,
    "args": {
      "External id": 2467730,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764256.814, "dur": 11.102,
    "args": {
      "External id": 2467731,"Record function id": 0, "Sequence number": 24740498, "Fwd thread id": 1, "Ev Idx": 1426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764258.008, "dur": 7.706,
    "args": {
      "External id": 2467732,"Sequence number": 24740498, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1427
    }
  },
  {
    "ph": "f", "id": 150, "pid": 1336756, "tid": 1381189, "ts": 1514656764258.008,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764259.542, "dur": 5.999,
    "args": {
      "External id": 2467733,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764262.438, "dur": 2.949,
    "args": {
      "External id": 2467734,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764271.141, "dur": 9.111,
    "args": {
      "External id": 2467735,"Record function id": 0, "Sequence number": 24740497, "Fwd thread id": 1, "Ev Idx": 1430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764272.033, "dur": 6.341,
    "args": {
      "External id": 2467736,"Sequence number": 24740497, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1431
    }
  },
  {
    "ph": "f", "id": 151, "pid": 1336756, "tid": 1381189, "ts": 1514656764272.033,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764273.528, "dur": 4.626,
    "args": {
      "External id": 2467737,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764274.277, "dur": 3.407,
    "args": {
      "External id": 2467738,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764277.098, "dur": 0.481,
    "args": {
      "External id": 2467739,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764283.953, "dur": 4.835,
    "args": {
      "External id": 2467740,"Record function id": 0, "Ev Idx": 1435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764285.176, "dur": 3.122,
    "args": {
      "External id": 2467741,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764286.413, "dur": 1.669,
    "args": {
      "External id": 2467742,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764286.870, "dur": 1.010,
    "args": {
      "External id": 2467743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764291.946, "dur": 6.394,
    "args": {
      "External id": 2467744,"Record function id": 0, "Sequence number": 24740496, "Fwd thread id": 1, "Ev Idx": 1439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764292.982, "dur": 3.457,
    "args": {
      "External id": 2467745,"Sequence number": 24740496, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1440
    }
  },
  {
    "ph": "f", "id": 152, "pid": 1336756, "tid": 1381189, "ts": 1514656764292.982,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764294.350, "dur": 1.936,
    "args": {
      "External id": 2467746,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764294.918, "dur": 1.208,
    "args": {
      "External id": 2467747,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764302.637, "dur": 310.591,
    "args": {
      "External id": 2467748,"Record function id": 0, "Sequence number": 24740495, "Fwd thread id": 1, "Ev Idx": 1443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764305.907, "dur": 284.135,
    "args": {
      "External id": 2467749,"Sequence number": 24740495, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1444
    }
  },
  {
    "ph": "f", "id": 153, "pid": 1336756, "tid": 1381189, "ts": 1514656764305.907,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764321.457, "dur": 9.045,
    "args": {
      "External id": 2467750,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764326.086, "dur": 3.967,
    "args": {
      "External id": 2467751,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764332.568, "dur": 5.165,
    "args": {
      "External id": 2467752,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764335.317, "dur": 2.188,
    "args": {
      "External id": 2467753,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764339.258, "dur": 2.634,
    "args": {
      "External id": 2467754,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764339.885, "dur": 1.734,
    "args": {
      "External id": 2467755,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764366.701, "dur": 196.232,
    "args": {
      "External id": 2467756,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764445.894, "dur": 4.394,
    "args": {
      "External id": 2467757,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764454.161, "dur": 4.039,
    "args": {
      "External id": 2467758,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764576.889, "dur": 3.991,
    "args": {
      "External id": 2467759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764584.119, "dur": 0.572,
    "args": {
      "External id": 2467760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764586.792, "dur": 0.740,
    "args": {
      "External id": 2467761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764622.663, "dur": 232.602,
    "args": {
      "External id": 2467762,"Record function id": 0, "Sequence number": 24740494, "Fwd thread id": 1, "Ev Idx": 1457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764624.404, "dur": 224.409,
    "args": {
      "External id": 2467763,"Sequence number": 24740494, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1458
    }
  },
  {
    "ph": "f", "id": 154, "pid": 1336756, "tid": 1381189, "ts": 1514656764624.404,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764645.246, "dur": 54.560,
    "args": {
      "External id": 2467764,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764649.948, "dur": 3.460,
    "args": {
      "External id": 2467765,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764654.905, "dur": 44.190,
    "args": {
      "External id": 2467766,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764709.200, "dur": 3.885,
    "args": {
      "External id": 2467767,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764710.499, "dur": 2.200,
    "args": {
      "External id": 2467768,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764862.065, "dur": 201.457,
    "args": {
      "External id": 2467769,"Record function id": 0, "Sequence number": 24740493, "Fwd thread id": 1, "Ev Idx": 1464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764865.817, "dur": 188.695,
    "args": {
      "External id": 2467770,"Sequence number": 24740493, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1465
    }
  },
  {
    "ph": "f", "id": 155, "pid": 1336756, "tid": 1381189, "ts": 1514656764865.817,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764878.771, "dur": 30.916,
    "args": {
      "External id": 2467771,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764880.706, "dur": 2.614,
    "args": {
      "External id": 2467772,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764884.383, "dur": 24.711,
    "args": {
      "External id": 2467773,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764916.295, "dur": 3.558,
    "args": {
      "External id": 2467774,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656764917.378, "dur": 2.166,
    "args": {
      "External id": 2467775,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765072.986, "dur": 16.757,
    "args": {
      "External id": 2467776,"Record function id": 0, "Sequence number": 24740492, "Fwd thread id": 1, "Ev Idx": 1471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765074.620, "dur": 12.667,
    "args": {
      "External id": 2467777,"Sequence number": 24740492, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1472
    }
  },
  {
    "ph": "f", "id": 156, "pid": 1336756, "tid": 1381189, "ts": 1514656765074.620,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765078.999, "dur": 7.998,
    "args": {
      "External id": 2467778,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765080.613, "dur": 6.233,
    "args": {
      "External id": 2467779,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765093.502, "dur": 7.272,
    "args": {
      "External id": 2467780,"Record function id": 0, "Sequence number": 24740491, "Fwd thread id": 1, "Ev Idx": 1475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765094.781, "dur": 3.381,
    "args": {
      "External id": 2467781,"Sequence number": 24740491, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1476
    }
  },
  {
    "ph": "f", "id": 157, "pid": 1336756, "tid": 1381189, "ts": 1514656765094.781,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765096.083, "dur": 1.937,
    "args": {
      "External id": 2467782,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765096.810, "dur": 1.045,
    "args": {
      "External id": 2467783,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765103.830, "dur": 10.174,
    "args": {
      "External id": 2467784,"Record function id": 0, "Sequence number": 24740490, "Fwd thread id": 1, "Ev Idx": 1479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765104.954, "dur": 7.017,
    "args": {
      "External id": 2467785,"Sequence number": 24740490, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1480
    }
  },
  {
    "ph": "f", "id": 158, "pid": 1336756, "tid": 1381189, "ts": 1514656765104.954,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765110.345, "dur": 1.478,
    "args": {
      "External id": 2467786,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765110.908, "dur": 0.807,
    "args": {
      "External id": 2467787,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765117.289, "dur": 6.398,
    "args": {
      "External id": 2467788,"Record function id": 0, "Sequence number": 24740489, "Fwd thread id": 1, "Ev Idx": 1483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765118.254, "dur": 3.106,
    "args": {
      "External id": 2467789,"Sequence number": 24740489, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1484
    }
  },
  {
    "ph": "f", "id": 159, "pid": 1336756, "tid": 1381189, "ts": 1514656765118.254,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765119.510, "dur": 1.707,
    "args": {
      "External id": 2467790,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765120.072, "dur": 1.018,
    "args": {
      "External id": 2467791,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765127.130, "dur": 149.337,
    "args": {
      "External id": 2467792,"Record function id": 0, "Sequence number": 24740488, "Fwd thread id": 1, "Ev Idx": 1487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765128.124, "dur": 141.200,
    "args": {
      "External id": 2467793,"Sequence number": 24740488, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1488
    }
  },
  {
    "ph": "f", "id": 160, "pid": 1336756, "tid": 1381189, "ts": 1514656765128.124,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765132.651, "dur": 10.443,
    "args": {
      "External id": 2467794,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765136.423, "dur": 6.055,
    "args": {
      "External id": 2467795,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765140.515, "dur": 1.674,
    "args": {
      "External id": 2467796,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765144.944, "dur": 66.377,
    "args": {
      "External id": 2467797,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765212.551, "dur": 4.212,
    "args": {
      "External id": 2467798,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765213.541, "dur": 2.486,
    "args": {
      "External id": 2467799,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765214.607, "dur": 1.254,
    "args": {
      "External id": 2467800,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765218.619, "dur": 5.158,
    "args": {
      "External id": 2467801,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765221.638, "dur": 1.416,
    "args": {
      "External id": 2467802,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765222.354, "dur": 0.624,
    "args": {
      "External id": 2467803,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765224.772, "dur": 43.543,
    "args": {
      "External id": 2467804,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765281.270, "dur": 8.019,
    "args": {
      "External id": 2467805,"Record function id": 0, "Sequence number": 24740487, "Fwd thread id": 1, "Ev Idx": 1500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765282.330, "dur": 5.311,
    "args": {
      "External id": 2467806,"Sequence number": 24740487, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1501
    }
  },
  {
    "ph": "f", "id": 161, "pid": 1336756, "tid": 1381189, "ts": 1514656765282.330,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765285.484, "dur": 2.012,
    "args": {
      "External id": 2467807,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765286.019, "dur": 1.378,
    "args": {
      "External id": 2467808,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765292.620, "dur": 9.836,
    "args": {
      "External id": 2467809,"Record function id": 0, "Sequence number": 24740486, "Fwd thread id": 1, "Ev Idx": 1504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765293.622, "dur": 6.051,
    "args": {
      "External id": 2467810,"Sequence number": 24740486, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1505
    }
  },
  {
    "ph": "f", "id": 162, "pid": 1336756, "tid": 1381189, "ts": 1514656765293.622,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765296.715, "dur": 2.760,
    "args": {
      "External id": 2467811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765297.601, "dur": 1.430,
    "args": {
      "External id": 2467812,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765298.463, "dur": 0.487,
    "args": {
      "External id": 2467813,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765308.250, "dur": 8.589,
    "args": {
      "External id": 2467814,"Record function id": 0, "Ev Idx": 1509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765309.733, "dur": 6.360,
    "args": {
      "External id": 2467815,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765311.976, "dur": 3.656,
    "args": {
      "External id": 2467816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765313.009, "dur": 2.530,
    "args": {
      "External id": 2467817,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765320.662, "dur": 9.755,
    "args": {
      "External id": 2467818,"Record function id": 0, "Sequence number": 24740485, "Fwd thread id": 1, "Ev Idx": 1513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765323.775, "dur": 4.985,
    "args": {
      "External id": 2467819,"Sequence number": 24740485, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1514
    }
  },
  {
    "ph": "f", "id": 163, "pid": 1336756, "tid": 1381189, "ts": 1514656765323.775,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765325.063, "dur": 3.541,
    "args": {
      "External id": 2467820,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765327.817, "dur": 0.680,
    "args": {
      "External id": 2467821,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765333.563, "dur": 94.196,
    "args": {
      "External id": 2467822,"Record function id": 0, "Sequence number": 24740484, "Fwd thread id": 1, "Ev Idx": 1517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765334.442, "dur": 86.679,
    "args": {
      "External id": 2467823,"Sequence number": 24740484, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1518
    }
  },
  {
    "ph": "f", "id": 164, "pid": 1336756, "tid": 1381189, "ts": 1514656765334.442,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765336.921, "dur": 2.384,
    "args": {
      "External id": 2467824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765337.642, "dur": 1.199,
    "args": {
      "External id": 2467825,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765338.257, "dur": 0.470,
    "args": {
      "External id": 2467826,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765340.200, "dur": 27.738,
    "args": {
      "External id": 2467827,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765370.840, "dur": 3.192,
    "args": {
      "External id": 2467828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765371.630, "dur": 1.882,
    "args": {
      "External id": 2467829,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765372.631, "dur": 0.730,
    "args": {
      "External id": 2467830,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765375.328, "dur": 7.721,
    "args": {
      "External id": 2467831,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765376.711, "dur": 5.760,
    "args": {
      "External id": 2467832,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765380.142, "dur": 2.238,
    "args": {
      "External id": 2467833,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765385.718, "dur": 34.663,
    "args": {
      "External id": 2467834,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765434.132, "dur": 35.316,
    "args": {
      "External id": 2467835,"Record function id": 0, "Sequence number": 24740483, "Fwd thread id": 1, "Ev Idx": 1530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765435.306, "dur": 7.416,
    "args": {
      "External id": 2467836,"Sequence number": 24740483, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1531
    }
  },
  {
    "ph": "f", "id": 165, "pid": 1336756, "tid": 1381189, "ts": 1514656765435.306,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765436.898, "dur": 5.675,
    "args": {
      "External id": 2467837,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765441.138, "dur": 1.337,
    "args": {
      "External id": 2467838,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765445.995, "dur": 21.118,
    "args": {
      "External id": 2467839,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765473.163, "dur": 35.596,
    "args": {
      "External id": 2467840,"Record function id": 0, "Sequence number": 24740482, "Fwd thread id": 1, "Ev Idx": 1535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765498.608, "dur": 8.223,
    "args": {
      "External id": 2467841,"Sequence number": 24740482, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1536
    }
  },
  {
    "ph": "f", "id": 166, "pid": 1336756, "tid": 1381189, "ts": 1514656765498.608,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765501.380, "dur": 5.227,
    "args": {
      "External id": 2467842,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765502.023, "dur": 3.960,
    "args": {
      "External id": 2467843,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765505.297, "dur": 0.530,
    "args": {
      "External id": 2467844,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765512.480, "dur": 4.795,
    "args": {
      "External id": 2467845,"Record function id": 0, "Ev Idx": 1540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765513.658, "dur": 3.082,
    "args": {
      "External id": 2467846,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765514.622, "dur": 1.627,
    "args": {
      "External id": 2467847,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765515.131, "dur": 1.012,
    "args": {
      "External id": 2467848,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765520.274, "dur": 29.541,
    "args": {
      "External id": 2467849,"Record function id": 0, "Sequence number": 24740481, "Fwd thread id": 1, "Ev Idx": 1544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765521.088, "dur": 5.050,
    "args": {
      "External id": 2467850,"Sequence number": 24740481, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1545
    }
  },
  {
    "ph": "f", "id": 167, "pid": 1336756, "tid": 1381189, "ts": 1514656765521.088,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765522.175, "dur": 3.835,
    "args": {
      "External id": 2467851,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765525.126, "dur": 0.795,
    "args": {
      "External id": 2467852,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765555.744, "dur": 105.726,
    "args": {
      "External id": 2467853,"Record function id": 0, "Sequence number": 24740480, "Fwd thread id": 1, "Ev Idx": 1548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765556.899, "dur": 94.973,
    "args": {
      "External id": 2467854,"Sequence number": 24740480, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1549
    }
  },
  {
    "ph": "f", "id": 168, "pid": 1336756, "tid": 1381189, "ts": 1514656765556.899,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765560.557, "dur": 4.888,
    "args": {
      "External id": 2467855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765561.160, "dur": 3.687,
    "args": {
      "External id": 2467856,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765564.195, "dur": 0.550,
    "args": {
      "External id": 2467857,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765566.144, "dur": 36.743,
    "args": {
      "External id": 2467858,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765606.487, "dur": 2.739,
    "args": {
      "External id": 2467859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765607.011, "dur": 1.681,
    "args": {
      "External id": 2467860,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765607.842, "dur": 0.720,
    "args": {
      "External id": 2467861,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765610.575, "dur": 2.510,
    "args": {
      "External id": 2467862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765611.699, "dur": 1.018,
    "args": {
      "External id": 2467863,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765612.297, "dur": 0.299,
    "args": {
      "External id": 2467864,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765615.939, "dur": 35.139,
    "args": {
      "External id": 2467865,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765665.965, "dur": 28.653,
    "args": {
      "External id": 2467866,"Record function id": 0, "Sequence number": 24740479, "Fwd thread id": 1, "Ev Idx": 1561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765666.895, "dur": 5.361,
    "args": {
      "External id": 2467867,"Sequence number": 24740479, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1562
    }
  },
  {
    "ph": "f", "id": 169, "pid": 1336756, "tid": 1381189, "ts": 1514656765666.895,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765670.332, "dur": 1.745,
    "args": {
      "External id": 2467868,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765670.852, "dur": 1.118,
    "args": {
      "External id": 2467869,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765674.715, "dur": 17.702,
    "args": {
      "External id": 2467870,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765698.389, "dur": 9.088,
    "args": {
      "External id": 2467871,"Record function id": 0, "Sequence number": 24740478, "Fwd thread id": 1, "Ev Idx": 1566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765699.245, "dur": 5.570,
    "args": {
      "External id": 2467872,"Sequence number": 24740478, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1567
    }
  },
  {
    "ph": "f", "id": 170, "pid": 1336756, "tid": 1381189, "ts": 1514656765699.245,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765700.129, "dur": 4.496,
    "args": {
      "External id": 2467873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765700.698, "dur": 3.436,
    "args": {
      "External id": 2467874,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765703.484, "dur": 0.526,
    "args": {
      "External id": 2467875,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765711.502, "dur": 4.914,
    "args": {
      "External id": 2467876,"Record function id": 0, "Ev Idx": 1571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765712.593, "dur": 3.319,
    "args": {
      "External id": 2467877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765713.566, "dur": 1.957,
    "args": {
      "External id": 2467878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765714.036, "dur": 1.378,
    "args": {
      "External id": 2467879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765720.496, "dur": 408.167,
    "args": {
      "External id": 2467880,"Record function id": 0, "Sequence number": 24740477, "Fwd thread id": 1, "Ev Idx": 1575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765724.325, "dur": 365.506,
    "args": {
      "External id": 2467881,"Sequence number": 24740477, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1576
    }
  },
  {
    "ph": "f", "id": 171, "pid": 1336756, "tid": 1381189, "ts": 1514656765724.325,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765757.164, "dur": 1.840,
    "args": {
      "External id": 2467882,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765757.713, "dur": 1.056,
    "args": {
      "External id": 2467883,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765775.762, "dur": 5.994,
    "args": {
      "External id": 2467884,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765791.414, "dur": 2.112,
    "args": {
      "External id": 2467885,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765942.669, "dur": 1.758,
    "args": {
      "External id": 2467886,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765948.353, "dur": 73.583,
    "args": {
      "External id": 2467887,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656765959.881, "dur": 0.747,
    "args": {
      "External id": 2467888,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766029.893, "dur": 35.781,
    "args": {
      "External id": 2467889,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766032.067, "dur": 33.415,
    "args": {
      "External id": 2467890,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766036.153, "dur": 5.111,
    "args": {
      "External id": 2467891,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766045.059, "dur": 19.806,
    "args": {
      "External id": 2467892,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766069.879, "dur": 4.889,
    "args": {
      "External id": 2467893,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766072.818, "dur": 1.830,
    "args": {
      "External id": 2467894,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766081.305, "dur": 1.783,
    "args": {
      "External id": 2467895,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766082.152, "dur": 0.833,
    "args": {
      "External id": 2467896,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766106.319, "dur": 17.116,
    "args": {
      "External id": 2467897,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766140.896, "dur": 11.533,
    "args": {
      "External id": 2467898,"Record function id": 0, "Ev Idx": 1593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766143.154, "dur": 8.431,
    "args": {
      "External id": 2467899,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766145.803, "dur": 4.942,
    "args": {
      "External id": 2467900,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766148.655, "dur": 1.992,
    "args": {
      "External id": 2467901,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766157.056, "dur": 2747.550,
    "args": {
      "External id": 2467902,"Record function id": 0, "Ev Idx": 1597
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.27)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766191.446, "dur": 945.941,
    "args": {
      "External id": 2467903,"Record function id": 0, "Ev Idx": 1598
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.26", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766219.169, "dur": 909.658,
    "args": {
      "External id": 2467904,"Record function id": 0, "Ev Idx": 1599
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766233.265, "dur": 880.250,
    "args": {
      "External id": 2467905,"Record function id": 0, "Ev Idx": 1600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766322.219, "dur": 6.687,
    "args": {
      "External id": 2467906,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766344.520, "dur": 37.294,
    "args": {
      "External id": 2467907,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766350.544, "dur": 3.494,
    "args": {
      "External id": 2467908,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766355.351, "dur": 0.753,
    "args": {
      "External id": 2467909,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766360.367, "dur": 0.451,
    "args": {
      "External id": 2467910,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766361.520, "dur": 0.321,
    "args": {
      "External id": 2467911,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766362.305, "dur": 0.436,
    "args": {
      "External id": 2467912,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766367.020, "dur": 0.262,
    "args": {
      "External id": 2467913,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766367.928, "dur": 0.379,
    "args": {
      "External id": 2467914,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766368.738, "dur": 2.277,
    "args": {
      "External id": 2467915,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766374.170, "dur": 2.143,
    "args": {
      "External id": 2467916,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766396.459, "dur": 33.686,
    "args": {
      "External id": 2467917,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766470.136, "dur": 138.824,
    "args": {
      "External id": 2467918,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766484.065, "dur": 5.666,
    "args": {
      "External id": 2467919,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766494.807, "dur": 9.385,
    "args": {
      "External id": 2467920,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766498.870, "dur": 4.963,
    "args": {
      "External id": 2467921,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766501.927, "dur": 0.750,
    "args": {
      "External id": 2467922,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766510.723, "dur": 51.094,
    "args": {
      "External id": 2467923,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766514.171, "dur": 0.823,
    "args": {
      "External id": 2467924,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766515.786, "dur": 2.174,
    "args": {
      "External id": 2467925,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766519.136, "dur": 0.552,
    "args": {
      "External id": 2467926,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766522.277, "dur": 0.413,
    "args": {
      "External id": 2467927,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766524.941, "dur": 0.241,
    "args": {
      "External id": 2467928,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766525.896, "dur": 19.356,
    "args": {
      "External id": 2467929,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766549.577, "dur": 0.414,
    "args": {
      "External id": 2467930,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766552.147, "dur": 0.529,
    "args": {
      "External id": 2467931,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766555.642, "dur": 0.363,
    "args": {
      "External id": 2467932,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766573.690, "dur": 27.249,
    "args": {
      "External id": 2467933,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766669.874, "dur": 301.347,
    "args": {
      "External id": 2467934,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766705.155, "dur": 261.436,
    "args": {
      "External id": 2467935,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1630, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514656766716.193, "dur": 245.180,
    "args": {
      "External id": 2467936,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767029.805, "dur": 3.570,
    "args": {
      "External id": 2467937,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1632, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767146.291, "dur": 1732.771,
    "args": {
      "External id": 2467938,"Sequence number": 24740476, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1633
    }
  },
  {
    "ph": "f", "id": 172, "pid": 1336756, "tid": 1381189, "ts": 1514656767146.291,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767277.043, "dur": 111.389,
    "args": {
      "External id": 2467939,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767434.487, "dur": 43.489,
    "args": {
      "External id": 2467940,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767497.857, "dur": 70.234,
    "args": {
      "External id": 2467941,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767581.785, "dur": 36.383,
    "args": {
      "External id": 2467942,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767624.961, "dur": 46.906,
    "args": {
      "External id": 2467943,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767680.610, "dur": 29.255,
    "args": {
      "External id": 2467944,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767717.373, "dur": 42.115,
    "args": {
      "External id": 2467945,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767789.289, "dur": 28.174,
    "args": {
      "External id": 2467946,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767837.683, "dur": 35.545,
    "args": {
      "External id": 2467947,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767898.762, "dur": 21.032,
    "args": {
      "External id": 2467948,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767935.606, "dur": 21.317,
    "args": {
      "External id": 2467949,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656767965.453, "dur": 64.974,
    "args": {
      "External id": 2467950,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768036.142, "dur": 37.961,
    "args": {
      "External id": 2467951,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768106.576, "dur": 177.533,
    "args": {
      "External id": 2467952,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768190.741, "dur": 7.462,
    "args": {
      "External id": 2467953,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768200.030, "dur": 1.960,
    "args": {
      "External id": 2467954,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768323.679, "dur": 26.919,
    "args": {
      "External id": 2467955,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768368.422, "dur": 28.618,
    "args": {
      "External id": 2467956,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768411.337, "dur": 39.220,
    "args": {
      "External id": 2467957,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768456.058, "dur": 35.387,
    "args": {
      "External id": 2467958,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768497.593, "dur": 24.344,
    "args": {
      "External id": 2467959,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768543.195, "dur": 38.975,
    "args": {
      "External id": 2467960,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768589.166, "dur": 20.174,
    "args": {
      "External id": 2467961,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768615.914, "dur": 29.001,
    "args": {
      "External id": 2467962,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768668.354, "dur": 26.838,
    "args": {
      "External id": 2467963,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768714.485, "dur": 26.896,
    "args": {
      "External id": 2467964,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768762.168, "dur": 18.434,
    "args": {
      "External id": 2467965,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768794.256, "dur": 15.190,
    "args": {
      "External id": 2467966,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768828.735, "dur": 20.352,
    "args": {
      "External id": 2467967,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768930.529, "dur": 15.515,
    "args": {
      "External id": 2467968,"Record function id": 0, "Ev Idx": 1663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768934.037, "dur": 11.201,
    "args": {
      "External id": 2467969,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768938.336, "dur": 5.770,
    "args": {
      "External id": 2467970,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768940.125, "dur": 3.869,
    "args": {
      "External id": 2467971,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768949.850, "dur": 8.267,
    "args": {
      "External id": 2467972,"Record function id": 0, "Ev Idx": 1667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768953.922, "dur": 3.681,
    "args": {
      "External id": 2467973,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768955.066, "dur": 1.858,
    "args": {
      "External id": 2467974,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768955.839, "dur": 1.009,
    "args": {
      "External id": 2467975,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768961.425, "dur": 5.272,
    "args": {
      "External id": 2467976,"Record function id": 0, "Ev Idx": 1671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768963.275, "dur": 2.975,
    "args": {
      "External id": 2467977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768964.071, "dur": 1.599,
    "args": {
      "External id": 2467978,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768964.960, "dur": 0.587,
    "args": {
      "External id": 2467979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768969.900, "dur": 7.061,
    "args": {
      "External id": 2467980,"Record function id": 0, "Ev Idx": 1675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768971.210, "dur": 5.323,
    "args": {
      "External id": 2467981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768972.090, "dur": 4.002,
    "args": {
      "External id": 2467982,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768972.628, "dur": 3.370,
    "args": {
      "External id": 2467983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768980.011, "dur": 3.878,
    "args": {
      "External id": 2467984,"Record function id": 0, "Ev Idx": 1679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768980.959, "dur": 2.495,
    "args": {
      "External id": 2467985,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768981.603, "dur": 1.228,
    "args": {
      "External id": 2467986,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656768982.032, "dur": 0.677,
    "args": {
      "External id": 2467987,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769022.914, "dur": 6.922,
    "args": {
      "External id": 2467988,"Record function id": 0, "Ev Idx": 1683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769024.844, "dur": 4.359,
    "args": {
      "External id": 2467989,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769026.096, "dur": 2.211,
    "args": {
      "External id": 2467990,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769026.646, "dur": 1.485,
    "args": {
      "External id": 2467991,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769033.165, "dur": 4.193,
    "args": {
      "External id": 2467992,"Record function id": 0, "Ev Idx": 1687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769034.567, "dur": 2.356,
    "args": {
      "External id": 2467993,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769035.108, "dur": 1.143,
    "args": {
      "External id": 2467994,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769035.520, "dur": 0.664,
    "args": {
      "External id": 2467995,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769040.436, "dur": 6.180,
    "args": {
      "External id": 2467996,"Record function id": 0, "Ev Idx": 1691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769041.680, "dur": 4.482,
    "args": {
      "External id": 2467997,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769042.288, "dur": 3.005,
    "args": {
      "External id": 2467998,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769044.656, "dur": 0.559,
    "args": {
      "External id": 2467999,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769050.683, "dur": 4.313,
    "args": {
      "External id": 2468000,"Record function id": 0, "Ev Idx": 1695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769051.970, "dur": 2.576,
    "args": {
      "External id": 2468001,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769052.483, "dur": 1.600,
    "args": {
      "External id": 2468002,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769052.920, "dur": 1.087,
    "args": {
      "External id": 2468003,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769059.170, "dur": 167020.276,
    "args": {
      "External id": 2468004,"Record function id": 0, "Sequence number": 24740475, "Fwd thread id": 1, "Ev Idx": 1699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769060.896, "dur": 167007.456,
    "args": {
      "External id": 2468005,"Sequence number": 24740475, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1700
    }
  },
  {
    "ph": "f", "id": 173, "pid": 1336756, "tid": 1381189, "ts": 1514656769060.896,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.27)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769092.915, "dur": 44.468,
    "args": {
      "External id": 2468006,"Record function id": 0, "Ev Idx": 1701
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.27)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769145.215, "dur": 91.108,
    "args": {
      "External id": 2468007,"Record function id": 0, "Ev Idx": 1702
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.27)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769243.377, "dur": 166814.842,
    "args": {
      "External id": 2468008,"Record function id": 0, "Ev Idx": 1703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769305.256, "dur": 7.477,
    "args": {
      "External id": 2468009,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769324.177, "dur": 4.968,
    "args": {
      "External id": 2468010,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769349.150, "dur": 165793.787,
    "args": {
      "External id": 2468011,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769363.804, "dur": 165768.964,
    "args": {
      "External id": 2468012,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769425.218, "dur": 5.478,
    "args": {
      "External id": 2468013,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769441.349, "dur": 165646.526,
    "args": {
      "External id": 2468014,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769443.629, "dur": 165643.328,
    "args": {
      "External id": 2468015,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769447.124, "dur": 6.996,
    "args": {
      "External id": 2468016,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656769455.866, "dur": 165626.962,
    "args": {
      "External id": 2468017,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935244.071, "dur": 10.046,
    "args": {
      "External id": 2468018,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935247.470, "dur": 6.280,
    "args": {
      "External id": 2468019,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935291.883, "dur": 387.899,
    "args": {
      "External id": 2468020,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935327.308, "dur": 347.131,
    "args": {
      "External id": 2468021,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1716, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935342.767, "dur": 326.186,
    "args": {
      "External id": 2468022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935708.740, "dur": 2.212,
    "args": {
      "External id": 2468023,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1718, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935776.447, "dur": 6.880,
    "args": {
      "External id": 2468024,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935838.325, "dur": 1.424,
    "args": {
      "External id": 2468025,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935857.250, "dur": 1.567,
    "args": {
      "External id": 2468026,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935875.723, "dur": 1.059,
    "args": {
      "External id": 2468027,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935888.821, "dur": 0.756,
    "args": {
      "External id": 2468028,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935900.510, "dur": 0.905,
    "args": {
      "External id": 2468029,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935914.821, "dur": 1.643,
    "args": {
      "External id": 2468030,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935930.495, "dur": 2.212,
    "args": {
      "External id": 2468031,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656935944.394, "dur": 0.844,
    "args": {
      "External id": 2468032,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936097.610, "dur": 2847.058,
    "args": {
      "External id": 2468033,"Record function id": 0, "Ev Idx": 1728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.26)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936118.696, "dur": 1103.125,
    "args": {
      "External id": 2468034,"Record function id": 0, "Ev Idx": 1729
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936133.620, "dur": 337.167,
    "args": {
      "External id": 2468035,"Record function id": 0, "Ev Idx": 1730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936224.729, "dur": 4.576,
    "args": {
      "External id": 2468036,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936232.339, "dur": 0.851,
    "args": {
      "External id": 2468037,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936234.776, "dur": 1.308,
    "args": {
      "External id": 2468038,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936237.508, "dur": 3.108,
    "args": {
      "External id": 2468039,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936243.782, "dur": 1.088,
    "args": {
      "External id": 2468040,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936246.373, "dur": 0.798,
    "args": {
      "External id": 2468041,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936248.524, "dur": 2.164,
    "args": {
      "External id": 2468042,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936252.019, "dur": 1.102,
    "args": {
      "External id": 2468043,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936256.845, "dur": 1.235,
    "args": {
      "External id": 2468044,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936259.368, "dur": 0.743,
    "args": {
      "External id": 2468045,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936278.429, "dur": 164.737,
    "args": {
      "External id": 2468046,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936297.433, "dur": 141.710,
    "args": {
      "External id": 2468047,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936322.083, "dur": 11.985,
    "args": {
      "External id": 2468048,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936337.998, "dur": 71.145,
    "args": {
      "External id": 2468049,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936342.787, "dur": 65.996,
    "args": {
      "External id": 2468050,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936346.476, "dur": 8.297,
    "args": {
      "External id": 2468051,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936356.592, "dur": 51.617,
    "args": {
      "External id": 2468052,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1747
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.25", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936579.988, "dur": 634.585,
    "args": {
      "External id": 2468053,"Record function id": 0, "Ev Idx": 1748
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936597.425, "dur": 604.637,
    "args": {
      "External id": 2468054,"Record function id": 0, "Ev Idx": 1749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936661.578, "dur": 5.370,
    "args": {
      "External id": 2468055,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936682.672, "dur": 37.777,
    "args": {
      "External id": 2468056,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936689.435, "dur": 3.134,
    "args": {
      "External id": 2468057,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936693.835, "dur": 0.621,
    "args": {
      "External id": 2468058,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936695.272, "dur": 0.492,
    "args": {
      "External id": 2468059,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936700.009, "dur": 0.493,
    "args": {
      "External id": 2468060,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936701.072, "dur": 0.513,
    "args": {
      "External id": 2468061,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936702.322, "dur": 0.546,
    "args": {
      "External id": 2468062,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936707.250, "dur": 2.121,
    "args": {
      "External id": 2468063,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936710.232, "dur": 0.410,
    "args": {
      "External id": 2468064,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936713.395, "dur": 2.121,
    "args": {
      "External id": 2468065,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936742.906, "dur": 36.972,
    "args": {
      "External id": 2468066,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936811.179, "dur": 107.065,
    "args": {
      "External id": 2468067,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936821.530, "dur": 4.169,
    "args": {
      "External id": 2468068,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936830.749, "dur": 12.228,
    "args": {
      "External id": 2468069,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936834.849, "dur": 7.693,
    "args": {
      "External id": 2468070,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936840.540, "dur": 0.848,
    "args": {
      "External id": 2468071,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936849.245, "dur": 27.489,
    "args": {
      "External id": 2468072,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936850.713, "dur": 0.609,
    "args": {
      "External id": 2468073,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936854.143, "dur": 2.658,
    "args": {
      "External id": 2468074,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936857.489, "dur": 0.332,
    "args": {
      "External id": 2468075,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936858.284, "dur": 2.321,
    "args": {
      "External id": 2468076,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936864.084, "dur": 0.324,
    "args": {
      "External id": 2468077,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936865.122, "dur": 0.737,
    "args": {
      "External id": 2468078,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936868.280, "dur": 0.510,
    "args": {
      "External id": 2468079,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936871.204, "dur": 0.295,
    "args": {
      "External id": 2468080,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936872.084, "dur": 0.287,
    "args": {
      "External id": 2468081,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936890.644, "dur": 20.160,
    "args": {
      "External id": 2468082,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656936959.630, "dur": 166.961,
    "args": {
      "External id": 2468083,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937024.883, "dur": 97.911,
    "args": {
      "External id": 2468084,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1779, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937037.542, "dur": 80.896,
    "args": {
      "External id": 2468085,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937145.915, "dur": 1.721,
    "args": {
      "External id": 2468086,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1781, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937228.713, "dur": 1693.144,
    "args": {
      "External id": 2468087,"Sequence number": 24740474, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1782
    }
  },
  {
    "ph": "f", "id": 174, "pid": 1336756, "tid": 1381189, "ts": 1514656937228.713,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937352.716, "dur": 105.285,
    "args": {
      "External id": 2468088,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937502.220, "dur": 60.077,
    "args": {
      "External id": 2468089,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937581.452, "dur": 56.087,
    "args": {
      "External id": 2468090,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937646.810, "dur": 33.104,
    "args": {
      "External id": 2468091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937687.520, "dur": 45.398,
    "args": {
      "External id": 2468092,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937739.428, "dur": 26.976,
    "args": {
      "External id": 2468093,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937774.222, "dur": 41.323,
    "args": {
      "External id": 2468094,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937841.455, "dur": 24.755,
    "args": {
      "External id": 2468095,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937888.684, "dur": 32.485,
    "args": {
      "External id": 2468096,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937941.618, "dur": 19.463,
    "args": {
      "External id": 2468097,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514656937978.038, "dur": 55.499,
    "args": {
      "External id": 2468098,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938044.666, "dur": 32.962,
    "args": {
      "External id": 2468099,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938080.863, "dur": 34.020,
    "args": {
      "External id": 2468100,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938145.318, "dur": 176.338,
    "args": {
      "External id": 2468101,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938230.154, "dur": 5.846,
    "args": {
      "External id": 2468102,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938237.960, "dur": 2.077,
    "args": {
      "External id": 2468103,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938362.542, "dur": 25.851,
    "args": {
      "External id": 2468104,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938401.774, "dur": 15.554,
    "args": {
      "External id": 2468105,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938425.237, "dur": 34.186,
    "args": {
      "External id": 2468106,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938464.787, "dur": 35.066,
    "args": {
      "External id": 2468107,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938510.246, "dur": 55.601,
    "args": {
      "External id": 2468108,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938580.985, "dur": 42.400,
    "args": {
      "External id": 2468109,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938630.946, "dur": 21.867,
    "args": {
      "External id": 2468110,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938659.546, "dur": 30.704,
    "args": {
      "External id": 2468111,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938716.860, "dur": 24.206,
    "args": {
      "External id": 2468112,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938760.935, "dur": 30.360,
    "args": {
      "External id": 2468113,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938808.347, "dur": 17.984,
    "args": {
      "External id": 2468114,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938843.338, "dur": 15.100,
    "args": {
      "External id": 2468115,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938874.213, "dur": 18.630,
    "args": {
      "External id": 2468116,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938967.105, "dur": 65.104,
    "args": {
      "External id": 2468117,"Record function id": 0, "Ev Idx": 1812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938970.416, "dur": 59.765,
    "args": {
      "External id": 2468118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938974.540, "dur": 5.917,
    "args": {
      "External id": 2468119,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656938976.055, "dur": 4.305,
    "args": {
      "External id": 2468120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939040.327, "dur": 6.998,
    "args": {
      "External id": 2468121,"Record function id": 0, "Ev Idx": 1816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939042.144, "dur": 4.746,
    "args": {
      "External id": 2468122,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939043.668, "dur": 2.707,
    "args": {
      "External id": 2468123,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939044.662, "dur": 1.625,
    "args": {
      "External id": 2468124,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939050.607, "dur": 5.318,
    "args": {
      "External id": 2468125,"Record function id": 0, "Ev Idx": 1820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939052.143, "dur": 3.353,
    "args": {
      "External id": 2468126,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939052.828, "dur": 1.909,
    "args": {
      "External id": 2468127,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939053.617, "dur": 1.002,
    "args": {
      "External id": 2468128,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939059.120, "dur": 6.173,
    "args": {
      "External id": 2468129,"Record function id": 0, "Ev Idx": 1824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939060.110, "dur": 4.761,
    "args": {
      "External id": 2468130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939061.034, "dur": 3.157,
    "args": {
      "External id": 2468131,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939061.456, "dur": 2.672,
    "args": {
      "External id": 2468132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939068.369, "dur": 3.690,
    "args": {
      "External id": 2468133,"Record function id": 0, "Ev Idx": 1828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939069.406, "dur": 2.263,
    "args": {
      "External id": 2468134,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939069.897, "dur": 1.324,
    "args": {
      "External id": 2468135,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939070.404, "dur": 0.757,
    "args": {
      "External id": 2468136,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939075.158, "dur": 5.121,
    "args": {
      "External id": 2468137,"Record function id": 0, "Ev Idx": 1832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939077.810, "dur": 2.029,
    "args": {
      "External id": 2468138,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939078.297, "dur": 1.110,
    "args": {
      "External id": 2468139,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939078.691, "dur": 0.644,
    "args": {
      "External id": 2468140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939083.585, "dur": 3.740,
    "args": {
      "External id": 2468141,"Record function id": 0, "Ev Idx": 1836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939084.741, "dur": 2.174,
    "args": {
      "External id": 2468142,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939085.302, "dur": 1.176,
    "args": {
      "External id": 2468143,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939085.632, "dur": 0.780,
    "args": {
      "External id": 2468144,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939090.368, "dur": 6.318,
    "args": {
      "External id": 2468145,"Record function id": 0, "Ev Idx": 1840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939091.624, "dur": 4.674,
    "args": {
      "External id": 2468146,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939092.245, "dur": 3.534,
    "args": {
      "External id": 2468147,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939094.628, "dur": 1.076,
    "args": {
      "External id": 2468148,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939099.706, "dur": 4.262,
    "args": {
      "External id": 2468149,"Record function id": 0, "Ev Idx": 1844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939101.267, "dur": 2.314,
    "args": {
      "External id": 2468150,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939102.008, "dur": 1.117,
    "args": {
      "External id": 2468151,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939102.294, "dur": 0.756,
    "args": {
      "External id": 2468152,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939108.044, "dur": 45065.184,
    "args": {
      "External id": 2468153,"Record function id": 0, "Sequence number": 24740473, "Fwd thread id": 1, "Ev Idx": 1848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939109.357, "dur": 45054.757,
    "args": {
      "External id": 2468154,"Sequence number": 24740473, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1849
    }
  },
  {
    "ph": "f", "id": 175, "pid": 1336756, "tid": 1381189, "ts": 1514656939109.357,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.26)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939141.043, "dur": 39.149,
    "args": {
      "External id": 2468155,"Record function id": 0, "Ev Idx": 1850
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.26)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939187.960, "dur": 65.283,
    "args": {
      "External id": 2468156,"Record function id": 0, "Ev Idx": 1851
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.26)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939260.971, "dur": 44894.590,
    "args": {
      "External id": 2468157,"Record function id": 0, "Ev Idx": 1852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939351.342, "dur": 7.094,
    "args": {
      "External id": 2468158,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939367.876, "dur": 4.923,
    "args": {
      "External id": 2468159,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939390.116, "dur": 43876.280,
    "args": {
      "External id": 2468160,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939403.292, "dur": 43854.433,
    "args": {
      "External id": 2468161,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939471.721, "dur": 15.028,
    "args": {
      "External id": 2468162,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939495.330, "dur": 43726.049,
    "args": {
      "External id": 2468163,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939497.601, "dur": 43723.026,
    "args": {
      "External id": 2468164,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939502.744, "dur": 5.079,
    "args": {
      "External id": 2468165,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656939509.610, "dur": 43707.429,
    "args": {
      "External id": 2468166,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983353.087, "dur": 8.065,
    "args": {
      "External id": 2468167,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983355.650, "dur": 5.075,
    "args": {
      "External id": 2468168,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983391.780, "dur": 416.326,
    "args": {
      "External id": 2468169,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983425.904, "dur": 376.923,
    "args": {
      "External id": 2468170,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1865, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983439.938, "dur": 356.671,
    "args": {
      "External id": 2468171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983832.710, "dur": 2.347,
    "args": {
      "External id": 2468172,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1867, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983896.462, "dur": 7.028,
    "args": {
      "External id": 2468173,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983949.078, "dur": 1.213,
    "args": {
      "External id": 2468174,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656983965.757, "dur": 1.158,
    "args": {
      "External id": 2468175,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984016.361, "dur": 1.691,
    "args": {
      "External id": 2468176,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984033.602, "dur": 0.876,
    "args": {
      "External id": 2468177,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984046.740, "dur": 0.745,
    "args": {
      "External id": 2468178,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984057.817, "dur": 0.972,
    "args": {
      "External id": 2468179,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984075.758, "dur": 2.290,
    "args": {
      "External id": 2468180,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984090.172, "dur": 0.894,
    "args": {
      "External id": 2468181,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984189.832, "dur": 2837.082,
    "args": {
      "External id": 2468182,"Record function id": 0, "Ev Idx": 1877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.25)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984210.163, "dur": 1068.040,
    "args": {
      "External id": 2468183,"Record function id": 0, "Ev Idx": 1878
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984224.808, "dur": 343.048,
    "args": {
      "External id": 2468184,"Record function id": 0, "Ev Idx": 1879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984310.079, "dur": 3.855,
    "args": {
      "External id": 2468185,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984316.878, "dur": 1.173,
    "args": {
      "External id": 2468186,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984319.674, "dur": 1.029,
    "args": {
      "External id": 2468187,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984322.253, "dur": 3.340,
    "args": {
      "External id": 2468188,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984329.267, "dur": 1.010,
    "args": {
      "External id": 2468189,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984331.706, "dur": 1.204,
    "args": {
      "External id": 2468190,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984336.399, "dur": 2.261,
    "args": {
      "External id": 2468191,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984340.012, "dur": 1.178,
    "args": {
      "External id": 2468192,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984344.201, "dur": 0.956,
    "args": {
      "External id": 2468193,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984346.321, "dur": 0.793,
    "args": {
      "External id": 2468194,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984364.056, "dur": 156.748,
    "args": {
      "External id": 2468195,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984381.597, "dur": 135.193,
    "args": {
      "External id": 2468196,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984400.057, "dur": 13.949,
    "args": {
      "External id": 2468197,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984417.911, "dur": 70.521,
    "args": {
      "External id": 2468198,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984422.920, "dur": 65.139,
    "args": {
      "External id": 2468199,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984426.141, "dur": 7.026,
    "args": {
      "External id": 2468200,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984434.925, "dur": 52.286,
    "args": {
      "External id": 2468201,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1896
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.24", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984659.154, "dur": 611.251,
    "args": {
      "External id": 2468202,"Record function id": 0, "Ev Idx": 1897
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984674.662, "dur": 583.563,
    "args": {
      "External id": 2468203,"Record function id": 0, "Ev Idx": 1898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984737.612, "dur": 5.560,
    "args": {
      "External id": 2468204,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984758.772, "dur": 32.824,
    "args": {
      "External id": 2468205,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984764.900, "dur": 1.464,
    "args": {
      "External id": 2468206,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984768.860, "dur": 0.714,
    "args": {
      "External id": 2468207,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984770.300, "dur": 0.598,
    "args": {
      "External id": 2468208,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984773.652, "dur": 1.258,
    "args": {
      "External id": 2468209,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984775.415, "dur": 0.601,
    "args": {
      "External id": 2468210,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984777.871, "dur": 0.351,
    "args": {
      "External id": 2468211,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984780.301, "dur": 2.433,
    "args": {
      "External id": 2468212,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984783.412, "dur": 0.392,
    "args": {
      "External id": 2468213,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984785.867, "dur": 0.490,
    "args": {
      "External id": 2468214,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984800.636, "dur": 35.072,
    "args": {
      "External id": 2468215,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984864.790, "dur": 115.737,
    "args": {
      "External id": 2468216,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984874.687, "dur": 2.808,
    "args": {
      "External id": 2468217,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984882.786, "dur": 13.109,
    "args": {
      "External id": 2468218,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984886.767, "dur": 8.675,
    "args": {
      "External id": 2468219,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984891.743, "dur": 2.427,
    "args": {
      "External id": 2468220,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984902.146, "dur": 38.397,
    "args": {
      "External id": 2468221,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984903.453, "dur": 0.874,
    "args": {
      "External id": 2468222,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984906.363, "dur": 0.653,
    "args": {
      "External id": 2468223,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984920.302, "dur": 0.628,
    "args": {
      "External id": 2468224,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984924.644, "dur": 2.567,
    "args": {
      "External id": 2468225,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984928.015, "dur": 0.392,
    "args": {
      "External id": 2468226,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984928.856, "dur": 1.317,
    "args": {
      "External id": 2468227,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984932.308, "dur": 0.248,
    "args": {
      "External id": 2468228,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984933.114, "dur": 0.306,
    "args": {
      "External id": 2468229,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984935.043, "dur": 0.296,
    "args": {
      "External id": 2468230,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656984951.332, "dur": 21.722,
    "args": {
      "External id": 2468231,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985064.178, "dur": 126.133,
    "args": {
      "External id": 2468232,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985095.459, "dur": 91.277,
    "args": {
      "External id": 2468233,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1928, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985104.587, "dur": 77.749,
    "args": {
      "External id": 2468234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985206.376, "dur": 1.966,
    "args": {
      "External id": 2468235,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1930, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985285.230, "dur": 1682.214,
    "args": {
      "External id": 2468236,"Sequence number": 24740472, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1931
    }
  },
  {
    "ph": "f", "id": 176, "pid": 1336756, "tid": 1381189, "ts": 1514656985285.230,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985396.860, "dur": 104.198,
    "args": {
      "External id": 2468237,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985564.296, "dur": 43.369,
    "args": {
      "External id": 2468238,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985626.476, "dur": 57.174,
    "args": {
      "External id": 2468239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985693.556, "dur": 33.463,
    "args": {
      "External id": 2468240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985734.854, "dur": 46.418,
    "args": {
      "External id": 2468241,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985787.639, "dur": 27.640,
    "args": {
      "External id": 2468242,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985831.690, "dur": 42.394,
    "args": {
      "External id": 2468243,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985901.244, "dur": 26.229,
    "args": {
      "External id": 2468244,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514656985947.380, "dur": 30.157,
    "args": {
      "External id": 2468245,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986041.072, "dur": 22.553,
    "args": {
      "External id": 2468246,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986083.216, "dur": 17.463,
    "args": {
      "External id": 2468247,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986109.929, "dur": 34.323,
    "args": {
      "External id": 2468248,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986147.857, "dur": 34.383,
    "args": {
      "External id": 2468249,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986210.725, "dur": 171.975,
    "args": {
      "External id": 2468250,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986291.188, "dur": 7.687,
    "args": {
      "External id": 2468251,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986300.841, "dur": 2.121,
    "args": {
      "External id": 2468252,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986416.913, "dur": 26.550,
    "args": {
      "External id": 2468253,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986455.706, "dur": 15.799,
    "args": {
      "External id": 2468254,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986479.474, "dur": 35.920,
    "args": {
      "External id": 2468255,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986520.248, "dur": 54.036,
    "args": {
      "External id": 2468256,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986604.428, "dur": 34.035,
    "args": {
      "External id": 2468257,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986643.488, "dur": 33.343,
    "args": {
      "External id": 2468258,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986683.249, "dur": 24.227,
    "args": {
      "External id": 2468259,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986715.108, "dur": 29.797,
    "args": {
      "External id": 2468260,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986771.703, "dur": 26.833,
    "args": {
      "External id": 2468261,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986818.178, "dur": 25.278,
    "args": {
      "External id": 2468262,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986857.891, "dur": 17.967,
    "args": {
      "External id": 2468263,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986892.355, "dur": 14.083,
    "args": {
      "External id": 2468264,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514656986922.760, "dur": 18.153,
    "args": {
      "External id": 2468265,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987049.813, "dur": 15.964,
    "args": {
      "External id": 2468266,"Record function id": 0, "Ev Idx": 1961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987053.266, "dur": 11.436,
    "args": {
      "External id": 2468267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987057.357, "dur": 6.062,
    "args": {
      "External id": 2468268,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987059.024, "dur": 4.295,
    "args": {
      "External id": 2468269,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987069.663, "dur": 6.055,
    "args": {
      "External id": 2468270,"Record function id": 0, "Ev Idx": 1965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987071.374, "dur": 3.909,
    "args": {
      "External id": 2468271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987072.196, "dur": 2.303,
    "args": {
      "External id": 2468272,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987073.110, "dur": 1.275,
    "args": {
      "External id": 2468273,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987078.856, "dur": 4.924,
    "args": {
      "External id": 2468274,"Record function id": 0, "Ev Idx": 1969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987080.222, "dur": 3.165,
    "args": {
      "External id": 2468275,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987080.905, "dur": 1.838,
    "args": {
      "External id": 2468276,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987081.637, "dur": 1.022,
    "args": {
      "External id": 2468277,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987086.867, "dur": 7.016,
    "args": {
      "External id": 2468278,"Record function id": 0, "Ev Idx": 1973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987088.701, "dur": 4.745,
    "args": {
      "External id": 2468279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987089.344, "dur": 3.526,
    "args": {
      "External id": 2468280,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987089.659, "dur": 3.149,
    "args": {
      "External id": 2468281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987096.918, "dur": 4.189,
    "args": {
      "External id": 2468282,"Record function id": 0, "Ev Idx": 1977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987098.200, "dur": 2.510,
    "args": {
      "External id": 2468283,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987098.699, "dur": 1.306,
    "args": {
      "External id": 2468284,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987098.991, "dur": 0.935,
    "args": {
      "External id": 2468285,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987104.166, "dur": 4.647,
    "args": {
      "External id": 2468286,"Record function id": 0, "Ev Idx": 1981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987105.748, "dur": 2.675,
    "args": {
      "External id": 2468287,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987106.467, "dur": 1.323,
    "args": {
      "External id": 2468288,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987107.148, "dur": 0.575,
    "args": {
      "External id": 2468289,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987112.201, "dur": 5.474,
    "args": {
      "External id": 2468290,"Record function id": 0, "Ev Idx": 1985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987113.327, "dur": 3.959,
    "args": {
      "External id": 2468291,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987113.833, "dur": 3.036,
    "args": {
      "External id": 2468292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987116.161, "dur": 0.635,
    "args": {
      "External id": 2468293,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987120.728, "dur": 3.757,
    "args": {
      "External id": 2468294,"Record function id": 0, "Ev Idx": 1989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987121.982, "dur": 2.121,
    "args": {
      "External id": 2468295,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987122.462, "dur": 1.030,
    "args": {
      "External id": 2468296,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987122.849, "dur": 0.562,
    "args": {
      "External id": 2468297,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987127.442, "dur": 3.527,
    "args": {
      "External id": 2468298,"Record function id": 0, "Ev Idx": 1993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987128.478, "dur": 2.100,
    "args": {
      "External id": 2468299,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987128.966, "dur": 0.973,
    "args": {
      "External id": 2468300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987129.221, "dur": 0.644,
    "args": {
      "External id": 2468301,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987134.402, "dur": 38887.494,
    "args": {
      "External id": 2468302,"Record function id": 0, "Sequence number": 24740471, "Fwd thread id": 1, "Ev Idx": 1997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987135.742, "dur": 38842.726,
    "args": {
      "External id": 2468303,"Sequence number": 24740471, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1998
    }
  },
  {
    "ph": "f", "id": 177, "pid": 1336756, "tid": 1381189, "ts": 1514656987135.742,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.25)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987167.033, "dur": 42.341,
    "args": {
      "External id": 2468304,"Record function id": 0, "Ev Idx": 1999
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.25)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987217.216, "dur": 64.380,
    "args": {
      "External id": 2468305,"Record function id": 0, "Ev Idx": 2000
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.25)", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987291.410, "dur": 38679.485,
    "args": {
      "External id": 2468306,"Record function id": 0, "Ev Idx": 2001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987380.866, "dur": 6.781,
    "args": {
      "External id": 2468307,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987397.184, "dur": 4.766,
    "args": {
      "External id": 2468308,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987416.972, "dur": 37744.363,
    "args": {
      "External id": 2468309,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987430.154, "dur": 37722.445,
    "args": {
      "External id": 2468310,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987480.431, "dur": 14.987,
    "args": {
      "External id": 2468311,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987502.020, "dur": 37613.064,
    "args": {
      "External id": 2468312,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987504.754, "dur": 37609.559,
    "args": {
      "External id": 2468313,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987508.878, "dur": 5.695,
    "args": {
      "External id": 2468314,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514656987516.463, "dur": 37594.168,
    "args": {
      "External id": 2468315,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025250.462, "dur": 9.096,
    "args": {
      "External id": 2468316,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025253.710, "dur": 5.459,
    "args": {
      "External id": 2468317,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025289.890, "dur": 373.331,
    "args": {
      "External id": 2468318,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025319.717, "dur": 338.331,
    "args": {
      "External id": 2468319,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2014, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025331.086, "dur": 320.905,
    "args": {
      "External id": 2468320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025687.036, "dur": 2.191,
    "args": {
      "External id": 2468321,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2016, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025749.811, "dur": 6.808,
    "args": {
      "External id": 2468322,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025803.132, "dur": 1.345,
    "args": {
      "External id": 2468323,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025822.336, "dur": 1.540,
    "args": {
      "External id": 2468324,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025839.817, "dur": 1.003,
    "args": {
      "External id": 2468325,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025853.613, "dur": 0.691,
    "args": {
      "External id": 2468326,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025866.677, "dur": 0.841,
    "args": {
      "External id": 2468327,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025880.455, "dur": 1.055,
    "args": {
      "External id": 2468328,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025894.840, "dur": 2.538,
    "args": {
      "External id": 2468329,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657025909.915, "dur": 0.884,
    "args": {
      "External id": 2468330,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026039.219, "dur": 2795.251,
    "args": {
      "External id": 2468331,"Record function id": 0, "Ev Idx": 2026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.24)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026061.007, "dur": 1100.320,
    "args": {
      "External id": 2468332,"Record function id": 0, "Ev Idx": 2027
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026075.483, "dur": 330.596,
    "args": {
      "External id": 2468333,"Record function id": 0, "Ev Idx": 2028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026161.166, "dur": 4.727,
    "args": {
      "External id": 2468334,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026169.103, "dur": 1.159,
    "args": {
      "External id": 2468335,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026174.496, "dur": 1.172,
    "args": {
      "External id": 2468336,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026177.283, "dur": 2.750,
    "args": {
      "External id": 2468337,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026181.655, "dur": 1.194,
    "args": {
      "External id": 2468338,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026184.229, "dur": 0.993,
    "args": {
      "External id": 2468339,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026188.586, "dur": 2.638,
    "args": {
      "External id": 2468340,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026193.074, "dur": 1.218,
    "args": {
      "External id": 2468341,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026195.854, "dur": 0.917,
    "args": {
      "External id": 2468342,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026198.369, "dur": 0.926,
    "args": {
      "External id": 2468343,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026220.409, "dur": 152.669,
    "args": {
      "External id": 2468344,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026236.817, "dur": 132.020,
    "args": {
      "External id": 2468345,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026258.161, "dur": 12.132,
    "args": {
      "External id": 2468346,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026273.659, "dur": 67.148,
    "args": {
      "External id": 2468347,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026276.182, "dur": 64.301,
    "args": {
      "External id": 2468348,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026280.477, "dur": 8.711,
    "args": {
      "External id": 2468349,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026290.995, "dur": 48.915,
    "args": {
      "External id": 2468350,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2045
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.23", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026496.752, "dur": 656.858,
    "args": {
      "External id": 2468351,"Record function id": 0, "Ev Idx": 2046
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026512.016, "dur": 628.870,
    "args": {
      "External id": 2468352,"Record function id": 0, "Ev Idx": 2047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026616.252, "dur": 6.318,
    "args": {
      "External id": 2468353,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026639.675, "dur": 32.574,
    "args": {
      "External id": 2468354,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026644.394, "dur": 3.042,
    "args": {
      "External id": 2468355,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026649.243, "dur": 0.509,
    "args": {
      "External id": 2468356,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026651.177, "dur": 0.318,
    "args": {
      "External id": 2468357,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026654.056, "dur": 0.329,
    "args": {
      "External id": 2468358,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026655.636, "dur": 0.425,
    "args": {
      "External id": 2468359,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026657.393, "dur": 0.339,
    "args": {
      "External id": 2468360,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026659.792, "dur": 2.154,
    "args": {
      "External id": 2468361,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026663.198, "dur": 0.318,
    "args": {
      "External id": 2468362,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026664.709, "dur": 1.432,
    "args": {
      "External id": 2468363,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026682.993, "dur": 37.368,
    "args": {
      "External id": 2468364,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026753.192, "dur": 101.779,
    "args": {
      "External id": 2468365,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026762.948, "dur": 3.422,
    "args": {
      "External id": 2468366,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026771.215, "dur": 9.926,
    "args": {
      "External id": 2468367,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026775.276, "dur": 5.440,
    "args": {
      "External id": 2468368,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026778.874, "dur": 0.486,
    "args": {
      "External id": 2468369,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026787.700, "dur": 26.527,
    "args": {
      "External id": 2468370,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026789.634, "dur": 0.330,
    "args": {
      "External id": 2468371,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026791.598, "dur": 0.380,
    "args": {
      "External id": 2468372,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026793.615, "dur": 1.198,
    "args": {
      "External id": 2468373,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026796.345, "dur": 2.447,
    "args": {
      "External id": 2468374,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026800.301, "dur": 0.331,
    "args": {
      "External id": 2468375,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026803.988, "dur": 0.292,
    "args": {
      "External id": 2468376,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026805.413, "dur": 0.331,
    "args": {
      "External id": 2468377,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026806.987, "dur": 0.291,
    "args": {
      "External id": 2468378,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026809.762, "dur": 0.311,
    "args": {
      "External id": 2468379,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026827.631, "dur": 19.636,
    "args": {
      "External id": 2468380,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026899.898, "dur": 162.723,
    "args": {
      "External id": 2468381,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026928.258, "dur": 130.180,
    "args": {
      "External id": 2468382,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2077, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657026937.946, "dur": 115.279,
    "args": {
      "External id": 2468383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027083.434, "dur": 1.944,
    "args": {
      "External id": 2468384,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2079, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027168.768, "dur": 1643.270,
    "args": {
      "External id": 2468385,"Sequence number": 24740470, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2080
    }
  },
  {
    "ph": "f", "id": 178, "pid": 1336756, "tid": 1381189, "ts": 1514657027168.768,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027280.438, "dur": 104.846,
    "args": {
      "External id": 2468386,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027428.705, "dur": 39.429,
    "args": {
      "External id": 2468387,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027482.945, "dur": 66.743,
    "args": {
      "External id": 2468388,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027562.787, "dur": 35.842,
    "args": {
      "External id": 2468389,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027606.773, "dur": 45.768,
    "args": {
      "External id": 2468390,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027659.510, "dur": 27.224,
    "args": {
      "External id": 2468391,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027693.734, "dur": 41.847,
    "args": {
      "External id": 2468392,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027760.103, "dur": 24.330,
    "args": {
      "External id": 2468393,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027807.920, "dur": 34.597,
    "args": {
      "External id": 2468394,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027864.506, "dur": 22.074,
    "args": {
      "External id": 2468395,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027900.887, "dur": 14.757,
    "args": {
      "External id": 2468396,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027924.255, "dur": 27.696,
    "args": {
      "External id": 2468397,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657027955.074, "dur": 68.241,
    "args": {
      "External id": 2468398,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028062.541, "dur": 172.771,
    "args": {
      "External id": 2468399,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028142.442, "dur": 5.396,
    "args": {
      "External id": 2468400,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028149.591, "dur": 2.030,
    "args": {
      "External id": 2468401,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028270.351, "dur": 25.255,
    "args": {
      "External id": 2468402,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028307.937, "dur": 15.560,
    "args": {
      "External id": 2468403,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028331.022, "dur": 40.730,
    "args": {
      "External id": 2468404,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028377.017, "dur": 50.819,
    "args": {
      "External id": 2468405,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028442.188, "dur": 25.272,
    "args": {
      "External id": 2468406,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028473.972, "dur": 32.326,
    "args": {
      "External id": 2468407,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028511.569, "dur": 36.124,
    "args": {
      "External id": 2468408,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028562.272, "dur": 32.790,
    "args": {
      "External id": 2468409,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028618.117, "dur": 24.471,
    "args": {
      "External id": 2468410,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028663.382, "dur": 23.371,
    "args": {
      "External id": 2468411,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028702.222, "dur": 17.225,
    "args": {
      "External id": 2468412,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028737.022, "dur": 15.693,
    "args": {
      "External id": 2468413,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028764.950, "dur": 18.613,
    "args": {
      "External id": 2468414,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028856.318, "dur": 16.172,
    "args": {
      "External id": 2468415,"Record function id": 0, "Ev Idx": 2110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028860.298, "dur": 11.347,
    "args": {
      "External id": 2468416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028864.641, "dur": 5.944,
    "args": {
      "External id": 2468417,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028866.480, "dur": 4.021,
    "args": {
      "External id": 2468418,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028876.392, "dur": 4.642,
    "args": {
      "External id": 2468419,"Record function id": 0, "Ev Idx": 2114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028877.824, "dur": 2.770,
    "args": {
      "External id": 2468420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028878.488, "dur": 1.526,
    "args": {
      "External id": 2468421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028879.025, "dur": 0.896,
    "args": {
      "External id": 2468422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028884.439, "dur": 4.796,
    "args": {
      "External id": 2468423,"Record function id": 0, "Ev Idx": 2118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028886.183, "dur": 2.668,
    "args": {
      "External id": 2468424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028886.691, "dur": 1.454,
    "args": {
      "External id": 2468425,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028887.415, "dur": 0.650,
    "args": {
      "External id": 2468426,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028892.342, "dur": 3.682,
    "args": {
      "External id": 2468427,"Record function id": 0, "Ev Idx": 2122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028893.573, "dur": 2.039,
    "args": {
      "External id": 2468428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028894.061, "dur": 1.134,
    "args": {
      "External id": 2468429,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028894.373, "dur": 0.749,
    "args": {
      "External id": 2468430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028899.261, "dur": 6.314,
    "args": {
      "External id": 2468431,"Record function id": 0, "Ev Idx": 2126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028900.363, "dur": 4.776,
    "args": {
      "External id": 2468432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028901.093, "dur": 3.607,
    "args": {
      "External id": 2468433,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028901.395, "dur": 3.241,
    "args": {
      "External id": 2468434,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028908.746, "dur": 4.500,
    "args": {
      "External id": 2468435,"Record function id": 0, "Ev Idx": 2130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028910.195, "dur": 2.646,
    "args": {
      "External id": 2468436,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028910.877, "dur": 1.377,
    "args": {
      "External id": 2468437,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028911.215, "dur": 0.976,
    "args": {
      "External id": 2468438,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028916.446, "dur": 5.573,
    "args": {
      "External id": 2468439,"Record function id": 0, "Ev Idx": 2134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028917.780, "dur": 3.815,
    "args": {
      "External id": 2468440,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028918.216, "dur": 2.793,
    "args": {
      "External id": 2468441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028920.357, "dur": 0.579,
    "args": {
      "External id": 2468442,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028925.162, "dur": 4.043,
    "args": {
      "External id": 2468443,"Record function id": 0, "Ev Idx": 2138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028926.544, "dur": 2.254,
    "args": {
      "External id": 2468444,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028927.118, "dur": 1.242,
    "args": {
      "External id": 2468445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028927.639, "dur": 0.656,
    "args": {
      "External id": 2468446,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028932.241, "dur": 3.919,
    "args": {
      "External id": 2468447,"Record function id": 0, "Ev Idx": 2142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028933.624, "dur": 2.146,
    "args": {
      "External id": 2468448,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028934.056, "dur": 1.010,
    "args": {
      "External id": 2468449,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028934.335, "dur": 0.657,
    "args": {
      "External id": 2468450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028939.783, "dur": 38152.822,
    "args": {
      "External id": 2468451,"Record function id": 0, "Sequence number": 24740469, "Fwd thread id": 1, "Ev Idx": 2146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028940.960, "dur": 38140.391,
    "args": {
      "External id": 2468452,"Sequence number": 24740469, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2147
    }
  },
  {
    "ph": "f", "id": 179, "pid": 1336756, "tid": 1381189, "ts": 1514657028940.960,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.24)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657028968.653, "dur": 74.373,
    "args": {
      "External id": 2468453,"Record function id": 0, "Ev Idx": 2148
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.24)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029052.413, "dur": 66.483,
    "args": {
      "External id": 2468454,"Record function id": 0, "Ev Idx": 2149
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.24)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029127.259, "dur": 37944.950,
    "args": {
      "External id": 2468455,"Record function id": 0, "Ev Idx": 2150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029213.589, "dur": 6.484,
    "args": {
      "External id": 2468456,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029230.960, "dur": 5.104,
    "args": {
      "External id": 2468457,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029250.098, "dur": 36911.084,
    "args": {
      "External id": 2468458,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029262.982, "dur": 36889.110,
    "args": {
      "External id": 2468459,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029310.588, "dur": 13.713,
    "args": {
      "External id": 2468460,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029330.610, "dur": 36783.570,
    "args": {
      "External id": 2468461,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029334.539, "dur": 36778.952,
    "args": {
      "External id": 2468462,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029338.591, "dur": 6.346,
    "args": {
      "External id": 2468463,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657029346.654, "dur": 36763.075,
    "args": {
      "External id": 2468464,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066253.896, "dur": 8.311,
    "args": {
      "External id": 2468465,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066256.859, "dur": 4.987,
    "args": {
      "External id": 2468466,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066294.299, "dur": 416.207,
    "args": {
      "External id": 2468467,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066324.011, "dur": 380.709,
    "args": {
      "External id": 2468468,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2163, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066334.556, "dur": 363.480,
    "args": {
      "External id": 2468469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066735.996, "dur": 2.531,
    "args": {
      "External id": 2468470,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2165, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066800.264, "dur": 6.824,
    "args": {
      "External id": 2468471,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066855.124, "dur": 1.298,
    "args": {
      "External id": 2468472,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066873.106, "dur": 1.209,
    "args": {
      "External id": 2468473,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066887.992, "dur": 2.976,
    "args": {
      "External id": 2468474,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066903.230, "dur": 0.738,
    "args": {
      "External id": 2468475,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066916.226, "dur": 0.734,
    "args": {
      "External id": 2468476,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066929.601, "dur": 0.965,
    "args": {
      "External id": 2468477,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066943.324, "dur": 3.948,
    "args": {
      "External id": 2468478,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657066960.372, "dur": 0.921,
    "args": {
      "External id": 2468479,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067109.219, "dur": 2811.864,
    "args": {
      "External id": 2468480,"Record function id": 0, "Ev Idx": 2175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.23)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067129.891, "dur": 1070.931,
    "args": {
      "External id": 2468481,"Record function id": 0, "Ev Idx": 2176
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067145.749, "dur": 323.899,
    "args": {
      "External id": 2468482,"Record function id": 0, "Ev Idx": 2177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067233.129, "dur": 4.517,
    "args": {
      "External id": 2468483,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067240.827, "dur": 1.166,
    "args": {
      "External id": 2468484,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067243.932, "dur": 1.090,
    "args": {
      "External id": 2468485,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067246.912, "dur": 0.572,
    "args": {
      "External id": 2468486,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067250.895, "dur": 3.348,
    "args": {
      "External id": 2468487,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067255.804, "dur": 1.088,
    "args": {
      "External id": 2468488,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067258.429, "dur": 1.881,
    "args": {
      "External id": 2468489,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067261.697, "dur": 1.032,
    "args": {
      "External id": 2468490,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067265.968, "dur": 1.109,
    "args": {
      "External id": 2468491,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067268.510, "dur": 0.917,
    "args": {
      "External id": 2468492,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067288.643, "dur": 148.933,
    "args": {
      "External id": 2468493,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067304.363, "dur": 129.068,
    "args": {
      "External id": 2468494,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067323.722, "dur": 14.103,
    "args": {
      "External id": 2468495,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067341.034, "dur": 65.266,
    "args": {
      "External id": 2468496,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067343.584, "dur": 62.374,
    "args": {
      "External id": 2468497,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067347.830, "dur": 4.648,
    "args": {
      "External id": 2468498,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067356.795, "dur": 48.532,
    "args": {
      "External id": 2468499,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2194
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.22", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067576.223, "dur": 616.895,
    "args": {
      "External id": 2468500,"Record function id": 0, "Ev Idx": 2195
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067594.533, "dur": 582.267,
    "args": {
      "External id": 2468501,"Record function id": 0, "Ev Idx": 2196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067657.916, "dur": 7.578,
    "args": {
      "External id": 2468502,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067680.985, "dur": 33.544,
    "args": {
      "External id": 2468503,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067686.244, "dur": 1.668,
    "args": {
      "External id": 2468504,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067691.464, "dur": 0.567,
    "args": {
      "External id": 2468505,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067693.277, "dur": 0.690,
    "args": {
      "External id": 2468506,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067695.743, "dur": 2.040,
    "args": {
      "External id": 2468507,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067699.387, "dur": 0.263,
    "args": {
      "External id": 2468508,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067701.023, "dur": 0.378,
    "args": {
      "External id": 2468509,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067704.310, "dur": 0.246,
    "args": {
      "External id": 2468510,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067705.776, "dur": 2.626,
    "args": {
      "External id": 2468511,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067709.716, "dur": 0.590,
    "args": {
      "External id": 2468512,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067724.602, "dur": 31.851,
    "args": {
      "External id": 2468513,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067786.960, "dur": 96.807,
    "args": {
      "External id": 2468514,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067796.672, "dur": 3.036,
    "args": {
      "External id": 2468515,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067804.737, "dur": 10.559,
    "args": {
      "External id": 2468516,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067808.925, "dur": 5.946,
    "args": {
      "External id": 2468517,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067812.578, "dur": 1.175,
    "args": {
      "External id": 2468518,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067822.010, "dur": 25.103,
    "args": {
      "External id": 2468519,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067824.021, "dur": 0.553,
    "args": {
      "External id": 2468520,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067826.080, "dur": 0.962,
    "args": {
      "External id": 2468521,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067828.178, "dur": 0.743,
    "args": {
      "External id": 2468522,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067831.144, "dur": 0.408,
    "args": {
      "External id": 2468523,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067832.852, "dur": 2.425,
    "args": {
      "External id": 2468524,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067836.773, "dur": 0.815,
    "args": {
      "External id": 2468525,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067838.709, "dur": 0.273,
    "args": {
      "External id": 2468526,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067840.153, "dur": 0.344,
    "args": {
      "External id": 2468527,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067842.836, "dur": 0.357,
    "args": {
      "External id": 2468528,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067856.481, "dur": 20.127,
    "args": {
      "External id": 2468529,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067925.666, "dur": 171.624,
    "args": {
      "External id": 2468530,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067953.417, "dur": 139.581,
    "args": {
      "External id": 2468531,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2226, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657067962.484, "dur": 125.811,
    "args": {
      "External id": 2468532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068117.807, "dur": 1.842,
    "args": {
      "External id": 2468533,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2228, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068208.952, "dur": 1689.296,
    "args": {
      "External id": 2468534,"Sequence number": 24740468, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2229
    }
  },
  {
    "ph": "f", "id": 180, "pid": 1336756, "tid": 1381189, "ts": 1514657068208.952,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068321.050, "dur": 103.309,
    "args": {
      "External id": 2468535,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068467.953, "dur": 39.263,
    "args": {
      "External id": 2468536,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068524.456, "dur": 73.313,
    "args": {
      "External id": 2468537,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068610.241, "dur": 33.005,
    "args": {
      "External id": 2468538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068649.881, "dur": 46.473,
    "args": {
      "External id": 2468539,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068707.686, "dur": 28.314,
    "args": {
      "External id": 2468540,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068743.874, "dur": 42.468,
    "args": {
      "External id": 2468541,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068811.786, "dur": 25.374,
    "args": {
      "External id": 2468542,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068854.817, "dur": 30.008,
    "args": {
      "External id": 2468543,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068907.564, "dur": 19.039,
    "args": {
      "External id": 2468544,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068940.616, "dur": 16.642,
    "args": {
      "External id": 2468545,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657068967.361, "dur": 68.467,
    "args": {
      "External id": 2468546,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069041.395, "dur": 37.697,
    "args": {
      "External id": 2468547,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069108.003, "dur": 179.523,
    "args": {
      "External id": 2468548,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069188.576, "dur": 6.287,
    "args": {
      "External id": 2468549,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069197.153, "dur": 4.286,
    "args": {
      "External id": 2468550,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069321.128, "dur": 27.531,
    "args": {
      "External id": 2468551,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069360.263, "dur": 16.886,
    "args": {
      "External id": 2468552,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069386.134, "dur": 36.672,
    "args": {
      "External id": 2468553,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069429.856, "dur": 36.753,
    "args": {
      "External id": 2468554,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069483.286, "dur": 23.300,
    "args": {
      "External id": 2468555,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069514.707, "dur": 69.814,
    "args": {
      "External id": 2468556,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069603.443, "dur": 26.961,
    "args": {
      "External id": 2468557,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069638.623, "dur": 35.565,
    "args": {
      "External id": 2468558,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069694.728, "dur": 27.360,
    "args": {
      "External id": 2468559,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069739.465, "dur": 27.298,
    "args": {
      "External id": 2468560,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069783.685, "dur": 17.651,
    "args": {
      "External id": 2468561,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069817.780, "dur": 19.683,
    "args": {
      "External id": 2468562,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069849.731, "dur": 17.818,
    "args": {
      "External id": 2468563,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069943.420, "dur": 15.613,
    "args": {
      "External id": 2468564,"Record function id": 0, "Ev Idx": 2259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069947.125, "dur": 10.860,
    "args": {
      "External id": 2468565,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069951.035, "dur": 5.956,
    "args": {
      "External id": 2468566,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069952.820, "dur": 4.053,
    "args": {
      "External id": 2468567,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069963.096, "dur": 5.112,
    "args": {
      "External id": 2468568,"Record function id": 0, "Ev Idx": 2263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069964.622, "dur": 3.113,
    "args": {
      "External id": 2468569,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069965.401, "dur": 1.731,
    "args": {
      "External id": 2468570,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069966.094, "dur": 0.958,
    "args": {
      "External id": 2468571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069971.601, "dur": 5.184,
    "args": {
      "External id": 2468572,"Record function id": 0, "Ev Idx": 2267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069973.184, "dur": 3.179,
    "args": {
      "External id": 2468573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069973.991, "dur": 1.749,
    "args": {
      "External id": 2468574,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069974.569, "dur": 1.092,
    "args": {
      "External id": 2468575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657069979.864, "dur": 39.384,
    "args": {
      "External id": 2468576,"Record function id": 0, "Ev Idx": 2271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070013.489, "dur": 4.688,
    "args": {
      "External id": 2468577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070014.849, "dur": 2.458,
    "args": {
      "External id": 2468578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070015.780, "dur": 1.270,
    "args": {
      "External id": 2468579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070024.078, "dur": 4.487,
    "args": {
      "External id": 2468580,"Record function id": 0, "Ev Idx": 2275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070025.813, "dur": 2.346,
    "args": {
      "External id": 2468581,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070026.450, "dur": 1.284,
    "args": {
      "External id": 2468582,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070026.882, "dur": 0.776,
    "args": {
      "External id": 2468583,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070031.761, "dur": 6.214,
    "args": {
      "External id": 2468584,"Record function id": 0, "Ev Idx": 2279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070032.912, "dur": 4.591,
    "args": {
      "External id": 2468585,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070033.792, "dur": 3.264,
    "args": {
      "External id": 2468586,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070034.262, "dur": 2.691,
    "args": {
      "External id": 2468587,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070041.483, "dur": 3.996,
    "args": {
      "External id": 2468588,"Record function id": 0, "Ev Idx": 2283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070042.773, "dur": 2.269,
    "args": {
      "External id": 2468589,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070043.277, "dur": 1.342,
    "args": {
      "External id": 2468590,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070043.770, "dur": 0.755,
    "args": {
      "External id": 2468591,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070048.539, "dur": 3.948,
    "args": {
      "External id": 2468592,"Record function id": 0, "Ev Idx": 2287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070049.961, "dur": 2.102,
    "args": {
      "External id": 2468593,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070050.447, "dur": 0.967,
    "args": {
      "External id": 2468594,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070050.695, "dur": 0.620,
    "args": {
      "External id": 2468595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070055.621, "dur": 4.891,
    "args": {
      "External id": 2468596,"Record function id": 0, "Ev Idx": 2291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070057.423, "dur": 2.677,
    "args": {
      "External id": 2468597,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070058.028, "dur": 1.339,
    "args": {
      "External id": 2468598,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070058.588, "dur": 0.666,
    "args": {
      "External id": 2468599,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070064.428, "dur": 37653.479,
    "args": {
      "External id": 2468600,"Record function id": 0, "Sequence number": 24740467, "Fwd thread id": 1, "Ev Idx": 2295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070065.879, "dur": 37643.448,
    "args": {
      "External id": 2468601,"Sequence number": 24740467, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2296
    }
  },
  {
    "ph": "f", "id": 181, "pid": 1336756, "tid": 1381189, "ts": 1514657070065.879,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.23)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070098.524, "dur": 40.780,
    "args": {
      "External id": 2468602,"Record function id": 0, "Ev Idx": 2297
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.23)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070147.610, "dur": 65.003,
    "args": {
      "External id": 2468603,"Record function id": 0, "Ev Idx": 2298
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.23)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070218.055, "dur": 37483.527,
    "args": {
      "External id": 2468604,"Record function id": 0, "Ev Idx": 2299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070308.146, "dur": 6.521,
    "args": {
      "External id": 2468605,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070324.288, "dur": 4.720,
    "args": {
      "External id": 2468606,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070344.400, "dur": 36515.752,
    "args": {
      "External id": 2468607,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070357.586, "dur": 36493.240,
    "args": {
      "External id": 2468608,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070404.712, "dur": 13.687,
    "args": {
      "External id": 2468609,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070424.832, "dur": 36385.573,
    "args": {
      "External id": 2468610,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070427.547, "dur": 36382.195,
    "args": {
      "External id": 2468611,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070432.003, "dur": 4.575,
    "args": {
      "External id": 2468612,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657070438.495, "dur": 36367.460,
    "args": {
      "External id": 2468613,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657106950.023, "dur": 8.440,
    "args": {
      "External id": 2468614,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657106953.158, "dur": 4.957,
    "args": {
      "External id": 2468615,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107015.069, "dur": 353.869,
    "args": {
      "External id": 2468616,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107046.357, "dur": 317.395,
    "args": {
      "External id": 2468617,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2312, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107057.871, "dur": 300.742,
    "args": {
      "External id": 2468618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107387.757, "dur": 1.990,
    "args": {
      "External id": 2468619,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2314, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107451.473, "dur": 6.639,
    "args": {
      "External id": 2468620,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107506.052, "dur": 1.394,
    "args": {
      "External id": 2468621,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107524.600, "dur": 0.948,
    "args": {
      "External id": 2468622,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107560.635, "dur": 1.404,
    "args": {
      "External id": 2468623,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107577.741, "dur": 0.853,
    "args": {
      "External id": 2468624,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107593.854, "dur": 0.991,
    "args": {
      "External id": 2468625,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107608.355, "dur": 0.820,
    "args": {
      "External id": 2468626,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107622.259, "dur": 2.245,
    "args": {
      "External id": 2468627,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107635.507, "dur": 0.968,
    "args": {
      "External id": 2468628,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107734.681, "dur": 2859.396,
    "args": {
      "External id": 2468629,"Record function id": 0, "Ev Idx": 2324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.22)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107753.891, "dur": 1068.742,
    "args": {
      "External id": 2468630,"Record function id": 0, "Ev Idx": 2325
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107769.162, "dur": 371.483,
    "args": {
      "External id": 2468631,"Record function id": 0, "Ev Idx": 2326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107854.865, "dur": 3.902,
    "args": {
      "External id": 2468632,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107862.133, "dur": 1.315,
    "args": {
      "External id": 2468633,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107865.440, "dur": 1.408,
    "args": {
      "External id": 2468634,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107868.529, "dur": 0.879,
    "args": {
      "External id": 2468635,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107871.165, "dur": 0.998,
    "args": {
      "External id": 2468636,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107873.833, "dur": 1.312,
    "args": {
      "External id": 2468637,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107876.826, "dur": 2.035,
    "args": {
      "External id": 2468638,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107880.353, "dur": 2.697,
    "args": {
      "External id": 2468639,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107884.554, "dur": 1.045,
    "args": {
      "External id": 2468640,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107887.257, "dur": 1.076,
    "args": {
      "External id": 2468641,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107907.264, "dur": 196.215,
    "args": {
      "External id": 2468642,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107922.871, "dur": 175.450,
    "args": {
      "External id": 2468643,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107943.776, "dur": 13.118,
    "args": {
      "External id": 2468644,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107960.515, "dur": 106.885,
    "args": {
      "External id": 2468645,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107962.933, "dur": 104.077,
    "args": {
      "External id": 2468646,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107967.025, "dur": 6.219,
    "args": {
      "External id": 2468647,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657107975.263, "dur": 90.343,
    "args": {
      "External id": 2468648,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2343
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.21", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108235.323, "dur": 578.793,
    "args": {
      "External id": 2468649,"Record function id": 0, "Ev Idx": 2344
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108254.044, "dur": 547.542,
    "args": {
      "External id": 2468650,"Record function id": 0, "Ev Idx": 2345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108316.393, "dur": 5.751,
    "args": {
      "External id": 2468651,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108338.010, "dur": 32.379,
    "args": {
      "External id": 2468652,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108343.136, "dur": 1.711,
    "args": {
      "External id": 2468653,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108347.224, "dur": 0.323,
    "args": {
      "External id": 2468654,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108349.350, "dur": 2.542,
    "args": {
      "External id": 2468655,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108353.952, "dur": 0.449,
    "args": {
      "External id": 2468656,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108356.159, "dur": 0.589,
    "args": {
      "External id": 2468657,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108358.762, "dur": 0.691,
    "args": {
      "External id": 2468658,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108361.350, "dur": 0.477,
    "args": {
      "External id": 2468659,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108363.672, "dur": 0.439,
    "args": {
      "External id": 2468660,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108366.008, "dur": 0.426,
    "args": {
      "External id": 2468661,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108385.398, "dur": 33.875,
    "args": {
      "External id": 2468662,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108453.229, "dur": 124.931,
    "args": {
      "External id": 2468663,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108464.291, "dur": 3.185,
    "args": {
      "External id": 2468664,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108472.625, "dur": 12.338,
    "args": {
      "External id": 2468665,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108476.932, "dur": 7.573,
    "args": {
      "External id": 2468666,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108480.727, "dur": 2.543,
    "args": {
      "External id": 2468667,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108492.054, "dur": 26.446,
    "args": {
      "External id": 2468668,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108494.623, "dur": 0.586,
    "args": {
      "External id": 2468669,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108497.015, "dur": 0.391,
    "args": {
      "External id": 2468670,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108499.421, "dur": 0.559,
    "args": {
      "External id": 2468671,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108501.782, "dur": 0.425,
    "args": {
      "External id": 2468672,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108503.746, "dur": 0.471,
    "args": {
      "External id": 2468673,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108505.751, "dur": 0.468,
    "args": {
      "External id": 2468674,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108508.074, "dur": 0.342,
    "args": {
      "External id": 2468675,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108510.141, "dur": 2.204,
    "args": {
      "External id": 2468676,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108514.554, "dur": 0.427,
    "args": {
      "External id": 2468677,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108545.641, "dur": 23.786,
    "args": {
      "External id": 2468678,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108623.868, "dur": 112.164,
    "args": {
      "External id": 2468679,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108646.752, "dur": 85.723,
    "args": {
      "External id": 2468680,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2375, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108657.008, "dur": 71.514,
    "args": {
      "External id": 2468681,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108750.682, "dur": 1.904,
    "args": {
      "External id": 2468682,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2377, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108831.429, "dur": 1739.337,
    "args": {
      "External id": 2468683,"Sequence number": 24740466, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2378
    }
  },
  {
    "ph": "f", "id": 182, "pid": 1336756, "tid": 1381189, "ts": 1514657108831.429,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657108942.164, "dur": 145.669,
    "args": {
      "External id": 2468684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109130.118, "dur": 41.449,
    "args": {
      "External id": 2468685,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109193.204, "dur": 56.270,
    "args": {
      "External id": 2468686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109259.948, "dur": 34.361,
    "args": {
      "External id": 2468687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109301.520, "dur": 46.872,
    "args": {
      "External id": 2468688,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109355.983, "dur": 28.642,
    "args": {
      "External id": 2468689,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109394.998, "dur": 43.391,
    "args": {
      "External id": 2468690,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109459.827, "dur": 23.671,
    "args": {
      "External id": 2468691,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109510.887, "dur": 50.941,
    "args": {
      "External id": 2468692,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109586.164, "dur": 23.381,
    "args": {
      "External id": 2468693,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109624.233, "dur": 17.859,
    "args": {
      "External id": 2468694,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109655.079, "dur": 34.411,
    "args": {
      "External id": 2468695,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109692.978, "dur": 34.945,
    "args": {
      "External id": 2468696,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109755.890, "dur": 178.594,
    "args": {
      "External id": 2468697,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109836.234, "dur": 5.997,
    "args": {
      "External id": 2468698,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109844.394, "dur": 2.654,
    "args": {
      "External id": 2468699,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657109967.910, "dur": 61.624,
    "args": {
      "External id": 2468700,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110044.080, "dur": 18.113,
    "args": {
      "External id": 2468701,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110075.349, "dur": 42.962,
    "args": {
      "External id": 2468702,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110125.638, "dur": 38.260,
    "args": {
      "External id": 2468703,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110172.165, "dur": 36.654,
    "args": {
      "External id": 2468704,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110220.082, "dur": 35.867,
    "args": {
      "External id": 2468705,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110265.171, "dur": 21.864,
    "args": {
      "External id": 2468706,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110295.654, "dur": 34.496,
    "args": {
      "External id": 2468707,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110349.362, "dur": 28.043,
    "args": {
      "External id": 2468708,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110394.283, "dur": 25.075,
    "args": {
      "External id": 2468709,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110435.049, "dur": 17.424,
    "args": {
      "External id": 2468710,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110467.117, "dur": 15.473,
    "args": {
      "External id": 2468711,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110504.985, "dur": 18.755,
    "args": {
      "External id": 2468712,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110616.735, "dur": 16.239,
    "args": {
      "External id": 2468713,"Record function id": 0, "Ev Idx": 2408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110620.664, "dur": 11.373,
    "args": {
      "External id": 2468714,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110625.228, "dur": 5.925,
    "args": {
      "External id": 2468715,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110626.899, "dur": 4.141,
    "args": {
      "External id": 2468716,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110636.850, "dur": 5.360,
    "args": {
      "External id": 2468717,"Record function id": 0, "Ev Idx": 2412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110638.426, "dur": 3.353,
    "args": {
      "External id": 2468718,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110639.311, "dur": 1.739,
    "args": {
      "External id": 2468719,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110639.950, "dur": 1.030,
    "args": {
      "External id": 2468720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110645.347, "dur": 5.276,
    "args": {
      "External id": 2468721,"Record function id": 0, "Ev Idx": 2416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110647.031, "dur": 3.188,
    "args": {
      "External id": 2468722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110647.806, "dur": 1.990,
    "args": {
      "External id": 2468723,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110648.326, "dur": 1.367,
    "args": {
      "External id": 2468724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110653.648, "dur": 4.723,
    "args": {
      "External id": 2468725,"Record function id": 0, "Ev Idx": 2420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110654.936, "dur": 3.034,
    "args": {
      "External id": 2468726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110655.962, "dur": 1.447,
    "args": {
      "External id": 2468727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110656.456, "dur": 0.879,
    "args": {
      "External id": 2468728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110661.389, "dur": 4.303,
    "args": {
      "External id": 2468729,"Record function id": 0, "Ev Idx": 2424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110663.058, "dur": 2.231,
    "args": {
      "External id": 2468730,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110663.729, "dur": 1.145,
    "args": {
      "External id": 2468731,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110664.078, "dur": 0.731,
    "args": {
      "External id": 2468732,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110669.243, "dur": 4.048,
    "args": {
      "External id": 2468733,"Record function id": 0, "Ev Idx": 2428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110670.403, "dur": 2.476,
    "args": {
      "External id": 2468734,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110670.855, "dur": 1.385,
    "args": {
      "External id": 2468735,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110671.276, "dur": 0.854,
    "args": {
      "External id": 2468736,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110676.708, "dur": 6.432,
    "args": {
      "External id": 2468737,"Record function id": 0, "Ev Idx": 2432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110678.187, "dur": 4.457,
    "args": {
      "External id": 2468738,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110678.851, "dur": 3.156,
    "args": {
      "External id": 2468739,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110679.138, "dur": 2.780,
    "args": {
      "External id": 2468740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110686.510, "dur": 4.133,
    "args": {
      "External id": 2468741,"Record function id": 0, "Ev Idx": 2436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110687.830, "dur": 2.411,
    "args": {
      "External id": 2468742,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110688.268, "dur": 1.558,
    "args": {
      "External id": 2468743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110688.721, "dur": 1.000,
    "args": {
      "External id": 2468744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110694.028, "dur": 3.909,
    "args": {
      "External id": 2468745,"Record function id": 0, "Ev Idx": 2440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110695.374, "dur": 2.162,
    "args": {
      "External id": 2468746,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110695.829, "dur": 1.157,
    "args": {
      "External id": 2468747,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110696.280, "dur": 0.606,
    "args": {
      "External id": 2468748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110701.514, "dur": 38007.427,
    "args": {
      "External id": 2468749,"Record function id": 0, "Sequence number": 24740465, "Fwd thread id": 1, "Ev Idx": 2444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110702.958, "dur": 37997.302,
    "args": {
      "External id": 2468750,"Sequence number": 24740465, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2445
    }
  },
  {
    "ph": "f", "id": 183, "pid": 1336756, "tid": 1381189, "ts": 1514657110702.958,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.22)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110733.526, "dur": 38.791,
    "args": {
      "External id": 2468751,"Record function id": 0, "Ev Idx": 2446
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.22)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110780.110, "dur": 66.757,
    "args": {
      "External id": 2468752,"Record function id": 0, "Ev Idx": 2447
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.22)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110852.805, "dur": 37839.551,
    "args": {
      "External id": 2468753,"Record function id": 0, "Ev Idx": 2448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110939.664, "dur": 6.626,
    "args": {
      "External id": 2468754,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110956.148, "dur": 4.393,
    "args": {
      "External id": 2468755,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657110975.503, "dur": 36844.985,
    "args": {
      "External id": 2468756,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657111025.083, "dur": 36785.599,
    "args": {
      "External id": 2468757,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657111078.372, "dur": 14.360,
    "args": {
      "External id": 2468758,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657111099.391, "dur": 36669.665,
    "args": {
      "External id": 2468759,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657111102.185, "dur": 36666.077,
    "args": {
      "External id": 2468760,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657111106.862, "dur": 5.706,
    "args": {
      "External id": 2468761,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657111114.634, "dur": 36649.893,
    "args": {
      "External id": 2468762,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657147911.456, "dur": 9.109,
    "args": {
      "External id": 2468763,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657147914.854, "dur": 5.341,
    "args": {
      "External id": 2468764,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657147952.507, "dur": 401.053,
    "args": {
      "External id": 2468765,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657147979.665, "dur": 367.511,
    "args": {
      "External id": 2468766,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2461, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148017.843, "dur": 322.508,
    "args": {
      "External id": 2468767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148376.192, "dur": 2.604,
    "args": {
      "External id": 2468768,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2463, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148444.277, "dur": 6.790,
    "args": {
      "External id": 2468769,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148501.502, "dur": 1.605,
    "args": {
      "External id": 2468770,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148520.616, "dur": 0.952,
    "args": {
      "External id": 2468771,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148552.518, "dur": 1.540,
    "args": {
      "External id": 2468772,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148569.261, "dur": 0.989,
    "args": {
      "External id": 2468773,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148583.865, "dur": 1.016,
    "args": {
      "External id": 2468774,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148596.772, "dur": 1.036,
    "args": {
      "External id": 2468775,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148611.419, "dur": 1.998,
    "args": {
      "External id": 2468776,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148626.052, "dur": 0.915,
    "args": {
      "External id": 2468777,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148724.844, "dur": 2895.515,
    "args": {
      "External id": 2468778,"Record function id": 0, "Ev Idx": 2473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.21)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148744.195, "dur": 1057.323,
    "args": {
      "External id": 2468779,"Record function id": 0, "Ev Idx": 2474
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148758.465, "dur": 371.117,
    "args": {
      "External id": 2468780,"Record function id": 0, "Ev Idx": 2475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148841.526, "dur": 4.105,
    "args": {
      "External id": 2468781,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148848.849, "dur": 1.360,
    "args": {
      "External id": 2468782,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148852.130, "dur": 1.469,
    "args": {
      "External id": 2468783,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148855.296, "dur": 1.633,
    "args": {
      "External id": 2468784,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148858.662, "dur": 1.272,
    "args": {
      "External id": 2468785,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148861.695, "dur": 1.072,
    "args": {
      "External id": 2468786,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148864.761, "dur": 2.029,
    "args": {
      "External id": 2468787,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148868.129, "dur": 3.448,
    "args": {
      "External id": 2468788,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148873.062, "dur": 1.047,
    "args": {
      "External id": 2468789,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148875.543, "dur": 0.890,
    "args": {
      "External id": 2468790,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148895.128, "dur": 197.489,
    "args": {
      "External id": 2468791,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148923.192, "dur": 164.006,
    "args": {
      "External id": 2468792,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148940.190, "dur": 11.795,
    "args": {
      "External id": 2468793,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148955.107, "dur": 100.921,
    "args": {
      "External id": 2468794,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148957.638, "dur": 97.925,
    "args": {
      "External id": 2468795,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148962.090, "dur": 6.017,
    "args": {
      "External id": 2468796,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657148970.263, "dur": 84.130,
    "args": {
      "External id": 2468797,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2492
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.20", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149221.415, "dur": 571.380,
    "args": {
      "External id": 2468798,"Record function id": 0, "Ev Idx": 2493
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149239.543, "dur": 540.670,
    "args": {
      "External id": 2468799,"Record function id": 0, "Ev Idx": 2494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149302.240, "dur": 5.694,
    "args": {
      "External id": 2468800,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149323.687, "dur": 30.784,
    "args": {
      "External id": 2468801,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149328.680, "dur": 1.561,
    "args": {
      "External id": 2468802,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149332.745, "dur": 0.416,
    "args": {
      "External id": 2468803,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149334.932, "dur": 2.453,
    "args": {
      "External id": 2468804,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149339.147, "dur": 0.603,
    "args": {
      "External id": 2468805,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149341.341, "dur": 0.351,
    "args": {
      "External id": 2468806,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149343.266, "dur": 0.439,
    "args": {
      "External id": 2468807,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149345.464, "dur": 0.432,
    "args": {
      "External id": 2468808,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149347.285, "dur": 0.423,
    "args": {
      "External id": 2468809,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149349.503, "dur": 0.469,
    "args": {
      "External id": 2468810,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149365.020, "dur": 32.460,
    "args": {
      "External id": 2468811,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149429.161, "dur": 117.041,
    "args": {
      "External id": 2468812,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149439.451, "dur": 3.383,
    "args": {
      "External id": 2468813,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149447.803, "dur": 11.502,
    "args": {
      "External id": 2468814,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149451.818, "dur": 7.038,
    "args": {
      "External id": 2468815,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149455.198, "dur": 2.495,
    "args": {
      "External id": 2468816,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149465.995, "dur": 26.047,
    "args": {
      "External id": 2468817,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149468.292, "dur": 0.539,
    "args": {
      "External id": 2468818,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149470.838, "dur": 0.424,
    "args": {
      "External id": 2468819,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149473.041, "dur": 0.474,
    "args": {
      "External id": 2468820,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149475.415, "dur": 0.496,
    "args": {
      "External id": 2468821,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149477.615, "dur": 0.392,
    "args": {
      "External id": 2468822,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149479.555, "dur": 0.367,
    "args": {
      "External id": 2468823,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149481.997, "dur": 0.642,
    "args": {
      "External id": 2468824,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149484.240, "dur": 1.935,
    "args": {
      "External id": 2468825,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149487.618, "dur": 0.491,
    "args": {
      "External id": 2468826,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149501.857, "dur": 20.366,
    "args": {
      "External id": 2468827,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149598.139, "dur": 115.208,
    "args": {
      "External id": 2468828,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149621.098, "dur": 88.941,
    "args": {
      "External id": 2468829,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2524, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149631.284, "dur": 74.140,
    "args": {
      "External id": 2468830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149727.437, "dur": 1.815,
    "args": {
      "External id": 2468831,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2526, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149808.387, "dur": 1785.101,
    "args": {
      "External id": 2468832,"Sequence number": 24740464, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2527
    }
  },
  {
    "ph": "f", "id": 184, "pid": 1336756, "tid": 1381189, "ts": 1514657149808.387,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657149919.710, "dur": 159.531,
    "args": {
      "External id": 2468833,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150123.393, "dur": 42.400,
    "args": {
      "External id": 2468834,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150187.664, "dur": 57.262,
    "args": {
      "External id": 2468835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150255.250, "dur": 35.170,
    "args": {
      "External id": 2468836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150298.030, "dur": 47.292,
    "args": {
      "External id": 2468837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150353.322, "dur": 28.491,
    "args": {
      "External id": 2468838,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150391.614, "dur": 42.788,
    "args": {
      "External id": 2468839,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150457.872, "dur": 24.053,
    "args": {
      "External id": 2468840,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150499.682, "dur": 46.009,
    "args": {
      "External id": 2468841,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150568.845, "dur": 21.594,
    "args": {
      "External id": 2468842,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150604.906, "dur": 16.596,
    "args": {
      "External id": 2468843,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150633.575, "dur": 32.796,
    "args": {
      "External id": 2468844,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150669.984, "dur": 34.818,
    "args": {
      "External id": 2468845,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150733.656, "dur": 181.780,
    "args": {
      "External id": 2468846,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150816.577, "dur": 6.065,
    "args": {
      "External id": 2468847,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150825.015, "dur": 2.774,
    "args": {
      "External id": 2468848,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657150970.000, "dur": 77.033,
    "args": {
      "External id": 2468849,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151062.295, "dur": 17.513,
    "args": {
      "External id": 2468850,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151093.788, "dur": 44.178,
    "args": {
      "External id": 2468851,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151145.496, "dur": 53.658,
    "args": {
      "External id": 2468852,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151212.336, "dur": 25.798,
    "args": {
      "External id": 2468853,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151243.975, "dur": 31.560,
    "args": {
      "External id": 2468854,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151288.024, "dur": 21.415,
    "args": {
      "External id": 2468855,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151318.255, "dur": 34.202,
    "args": {
      "External id": 2468856,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151370.812, "dur": 24.467,
    "args": {
      "External id": 2468857,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151412.632, "dur": 28.159,
    "args": {
      "External id": 2468858,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151457.076, "dur": 19.404,
    "args": {
      "External id": 2468859,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151493.930, "dur": 15.455,
    "args": {
      "External id": 2468860,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151522.275, "dur": 36.823,
    "args": {
      "External id": 2468861,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151643.684, "dur": 40.677,
    "args": {
      "External id": 2468862,"Record function id": 0, "Ev Idx": 2557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151647.340, "dur": 36.088,
    "args": {
      "External id": 2468863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151651.467, "dur": 30.943,
    "args": {
      "External id": 2468864,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151677.969, "dur": 4.292,
    "args": {
      "External id": 2468865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151688.507, "dur": 5.283,
    "args": {
      "External id": 2468866,"Record function id": 0, "Ev Idx": 2561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151690.132, "dur": 3.204,
    "args": {
      "External id": 2468867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151690.930, "dur": 1.840,
    "args": {
      "External id": 2468868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151691.636, "dur": 1.040,
    "args": {
      "External id": 2468869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151697.392, "dur": 4.240,
    "args": {
      "External id": 2468870,"Record function id": 0, "Ev Idx": 2565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151698.716, "dur": 2.505,
    "args": {
      "External id": 2468871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151699.390, "dur": 1.397,
    "args": {
      "External id": 2468872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151699.809, "dur": 0.893,
    "args": {
      "External id": 2468873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151705.159, "dur": 3.631,
    "args": {
      "External id": 2468874,"Record function id": 0, "Ev Idx": 2569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151706.369, "dur": 2.034,
    "args": {
      "External id": 2468875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151706.863, "dur": 0.998,
    "args": {
      "External id": 2468876,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151707.169, "dur": 0.630,
    "args": {
      "External id": 2468877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151712.096, "dur": 3.652,
    "args": {
      "External id": 2468878,"Record function id": 0, "Ev Idx": 2573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151713.174, "dur": 2.166,
    "args": {
      "External id": 2468879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151713.886, "dur": 1.036,
    "args": {
      "External id": 2468880,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151714.175, "dur": 0.680,
    "args": {
      "External id": 2468881,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151719.225, "dur": 4.150,
    "args": {
      "External id": 2468882,"Record function id": 0, "Ev Idx": 2577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151720.443, "dur": 2.549,
    "args": {
      "External id": 2468883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151720.939, "dur": 1.299,
    "args": {
      "External id": 2468884,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151721.398, "dur": 0.768,
    "args": {
      "External id": 2468885,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151726.623, "dur": 5.970,
    "args": {
      "External id": 2468886,"Record function id": 0, "Ev Idx": 2581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151727.845, "dur": 4.324,
    "args": {
      "External id": 2468887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151728.502, "dur": 2.885,
    "args": {
      "External id": 2468888,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151728.792, "dur": 2.509,
    "args": {
      "External id": 2468889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151735.815, "dur": 5.652,
    "args": {
      "External id": 2468890,"Record function id": 0, "Ev Idx": 2585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151737.189, "dur": 3.890,
    "args": {
      "External id": 2468891,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151737.683, "dur": 2.717,
    "args": {
      "External id": 2468892,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151739.530, "dur": 0.757,
    "args": {
      "External id": 2468893,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151744.638, "dur": 3.618,
    "args": {
      "External id": 2468894,"Record function id": 0, "Ev Idx": 2589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151745.771, "dur": 2.053,
    "args": {
      "External id": 2468895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151746.248, "dur": 1.124,
    "args": {
      "External id": 2468896,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151746.683, "dur": 0.601,
    "args": {
      "External id": 2468897,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151752.752, "dur": 39392.396,
    "args": {
      "External id": 2468898,"Record function id": 0, "Sequence number": 24740463, "Fwd thread id": 1, "Ev Idx": 2593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151753.920, "dur": 39382.082,
    "args": {
      "External id": 2468899,"Sequence number": 24740463, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2594
    }
  },
  {
    "ph": "f", "id": 185, "pid": 1336756, "tid": 1381189, "ts": 1514657151753.920,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.21)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151784.966, "dur": 38.452,
    "args": {
      "External id": 2468900,"Record function id": 0, "Ev Idx": 2595
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.21)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151830.902, "dur": 67.194,
    "args": {
      "External id": 2468901,"Record function id": 0, "Ev Idx": 2596
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.21)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657151903.656, "dur": 39222.995,
    "args": {
      "External id": 2468902,"Record function id": 0, "Ev Idx": 2597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152037.698, "dur": 8.411,
    "args": {
      "External id": 2468903,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152057.627, "dur": 5.275,
    "args": {
      "External id": 2468904,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152084.416, "dur": 38136.283,
    "args": {
      "External id": 2468905,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152097.832, "dur": 38114.000,
    "args": {
      "External id": 2468906,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152143.048, "dur": 14.735,
    "args": {
      "External id": 2468907,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152164.169, "dur": 38007.187,
    "args": {
      "External id": 2468908,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152166.747, "dur": 38003.980,
    "args": {
      "External id": 2468909,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152171.020, "dur": 5.309,
    "args": {
      "External id": 2468910,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657152178.317, "dur": 37988.943,
    "args": {
      "External id": 2468911,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190310.841, "dur": 9.167,
    "args": {
      "External id": 2468912,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190314.254, "dur": 5.381,
    "args": {
      "External id": 2468913,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190351.609, "dur": 423.053,
    "args": {
      "External id": 2468914,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190378.300, "dur": 390.926,
    "args": {
      "External id": 2468915,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2610, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190389.196, "dur": 373.866,
    "args": {
      "External id": 2468916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190797.485, "dur": 2.250,
    "args": {
      "External id": 2468917,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2612, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190861.216, "dur": 8.693,
    "args": {
      "External id": 2468918,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190918.767, "dur": 1.326,
    "args": {
      "External id": 2468919,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190937.664, "dur": 1.052,
    "args": {
      "External id": 2468920,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190952.375, "dur": 0.993,
    "args": {
      "External id": 2468921,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190965.945, "dur": 3.025,
    "args": {
      "External id": 2468922,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657190980.849, "dur": 33.152,
    "args": {
      "External id": 2468923,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191032.698, "dur": 1.322,
    "args": {
      "External id": 2468924,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191047.514, "dur": 1.959,
    "args": {
      "External id": 2468925,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191061.366, "dur": 2.321,
    "args": {
      "External id": 2468926,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191161.462, "dur": 2817.637,
    "args": {
      "External id": 2468927,"Record function id": 0, "Ev Idx": 2622
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.20)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191185.308, "dur": 1045.773,
    "args": {
      "External id": 2468928,"Record function id": 0, "Ev Idx": 2623
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191201.295, "dur": 324.591,
    "args": {
      "External id": 2468929,"Record function id": 0, "Ev Idx": 2624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191288.516, "dur": 3.723,
    "args": {
      "External id": 2468930,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191295.532, "dur": 1.239,
    "args": {
      "External id": 2468931,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191298.629, "dur": 1.313,
    "args": {
      "External id": 2468932,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191301.638, "dur": 1.095,
    "args": {
      "External id": 2468933,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191304.292, "dur": 1.012,
    "args": {
      "External id": 2468934,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191307.083, "dur": 0.964,
    "args": {
      "External id": 2468935,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191309.497, "dur": 3.537,
    "args": {
      "External id": 2468936,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191314.485, "dur": 1.023,
    "args": {
      "External id": 2468937,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191317.389, "dur": 0.883,
    "args": {
      "External id": 2468938,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191319.747, "dur": 0.781,
    "args": {
      "External id": 2468939,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191341.136, "dur": 151.822,
    "args": {
      "External id": 2468940,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191357.117, "dur": 131.923,
    "args": {
      "External id": 2468941,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191378.887, "dur": 11.913,
    "args": {
      "External id": 2468942,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191394.036, "dur": 66.377,
    "args": {
      "External id": 2468943,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191396.587, "dur": 63.518,
    "args": {
      "External id": 2468944,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191401.014, "dur": 5.974,
    "args": {
      "External id": 2468945,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191409.090, "dur": 50.329,
    "args": {
      "External id": 2468946,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2641
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.19", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191635.197, "dur": 587.534,
    "args": {
      "External id": 2468947,"Record function id": 0, "Ev Idx": 2642
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191655.103, "dur": 554.035,
    "args": {
      "External id": 2468948,"Record function id": 0, "Ev Idx": 2643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191718.375, "dur": 5.280,
    "args": {
      "External id": 2468949,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191739.446, "dur": 32.408,
    "args": {
      "External id": 2468950,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191744.727, "dur": 1.545,
    "args": {
      "External id": 2468951,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191748.681, "dur": 2.440,
    "args": {
      "External id": 2468952,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191753.322, "dur": 0.467,
    "args": {
      "External id": 2468953,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191755.498, "dur": 0.493,
    "args": {
      "External id": 2468954,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191758.465, "dur": 0.399,
    "args": {
      "External id": 2468955,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191760.303, "dur": 0.415,
    "args": {
      "External id": 2468956,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191762.692, "dur": 0.367,
    "args": {
      "External id": 2468957,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191764.869, "dur": 0.410,
    "args": {
      "External id": 2468958,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191766.995, "dur": 0.541,
    "args": {
      "External id": 2468959,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191781.900, "dur": 33.519,
    "args": {
      "External id": 2468960,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191846.131, "dur": 102.795,
    "args": {
      "External id": 2468961,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191856.233, "dur": 5.194,
    "args": {
      "External id": 2468962,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191866.685, "dur": 9.934,
    "args": {
      "External id": 2468963,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191870.803, "dur": 5.396,
    "args": {
      "External id": 2468964,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191874.364, "dur": 0.682,
    "args": {
      "External id": 2468965,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191883.678, "dur": 26.288,
    "args": {
      "External id": 2468966,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191886.402, "dur": 0.659,
    "args": {
      "External id": 2468967,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191889.091, "dur": 0.431,
    "args": {
      "External id": 2468968,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191891.071, "dur": 0.532,
    "args": {
      "External id": 2468969,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191893.133, "dur": 0.399,
    "args": {
      "External id": 2468970,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191894.955, "dur": 0.413,
    "args": {
      "External id": 2468971,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191897.015, "dur": 0.369,
    "args": {
      "External id": 2468972,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191899.406, "dur": 2.234,
    "args": {
      "External id": 2468973,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191903.736, "dur": 0.341,
    "args": {
      "External id": 2468974,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191906.192, "dur": 0.541,
    "args": {
      "External id": 2468975,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657191921.075, "dur": 20.541,
    "args": {
      "External id": 2468976,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192028.481, "dur": 114.458,
    "args": {
      "External id": 2468977,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192051.195, "dur": 88.091,
    "args": {
      "External id": 2468978,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2673, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192062.204, "dur": 72.791,
    "args": {
      "External id": 2468979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192156.155, "dur": 1.938,
    "args": {
      "External id": 2468980,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2675, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192238.842, "dur": 1718.720,
    "args": {
      "External id": 2468981,"Sequence number": 24740462, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2676
    }
  },
  {
    "ph": "f", "id": 186, "pid": 1336756, "tid": 1381189, "ts": 1514657192238.842,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192349.803, "dur": 103.559,
    "args": {
      "External id": 2468982,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192492.035, "dur": 58.583,
    "args": {
      "External id": 2468983,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192575.412, "dur": 59.475,
    "args": {
      "External id": 2468984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192644.940, "dur": 35.086,
    "args": {
      "External id": 2468985,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192688.272, "dur": 47.967,
    "args": {
      "External id": 2468986,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192746.825, "dur": 28.997,
    "args": {
      "External id": 2468987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192784.286, "dur": 44.252,
    "args": {
      "External id": 2468988,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192853.106, "dur": 25.101,
    "args": {
      "External id": 2468989,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192899.089, "dur": 31.022,
    "args": {
      "External id": 2468990,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657192951.775, "dur": 21.073,
    "args": {
      "External id": 2468991,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193036.010, "dur": 21.382,
    "args": {
      "External id": 2468992,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193072.359, "dur": 36.504,
    "args": {
      "External id": 2468993,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193112.998, "dur": 37.138,
    "args": {
      "External id": 2468994,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193179.948, "dur": 180.000,
    "args": {
      "External id": 2468995,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193260.080, "dur": 6.720,
    "args": {
      "External id": 2468996,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193269.154, "dur": 4.244,
    "args": {
      "External id": 2468997,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193395.242, "dur": 24.361,
    "args": {
      "External id": 2468998,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193431.599, "dur": 17.507,
    "args": {
      "External id": 2468999,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193457.732, "dur": 35.729,
    "args": {
      "External id": 2469000,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193500.154, "dur": 69.501,
    "args": {
      "External id": 2469001,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193589.352, "dur": 32.508,
    "args": {
      "External id": 2469002,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193629.578, "dur": 35.333,
    "args": {
      "External id": 2469003,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193672.494, "dur": 24.637,
    "args": {
      "External id": 2469004,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193705.616, "dur": 32.749,
    "args": {
      "External id": 2469005,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193761.586, "dur": 24.504,
    "args": {
      "External id": 2469006,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193805.576, "dur": 24.650,
    "args": {
      "External id": 2469007,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193845.422, "dur": 21.834,
    "args": {
      "External id": 2469008,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193882.450, "dur": 15.630,
    "args": {
      "External id": 2469009,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657193911.982, "dur": 17.585,
    "args": {
      "External id": 2469010,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194033.226, "dur": 17.623,
    "args": {
      "External id": 2469011,"Record function id": 0, "Ev Idx": 2706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194037.162, "dur": 12.283,
    "args": {
      "External id": 2469012,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194041.644, "dur": 6.166,
    "args": {
      "External id": 2469013,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194043.334, "dur": 4.198,
    "args": {
      "External id": 2469014,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194055.154, "dur": 5.450,
    "args": {
      "External id": 2469015,"Record function id": 0, "Ev Idx": 2710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194056.516, "dur": 3.671,
    "args": {
      "External id": 2469016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194057.472, "dur": 2.236,
    "args": {
      "External id": 2469017,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194058.275, "dur": 1.290,
    "args": {
      "External id": 2469018,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194064.125, "dur": 4.746,
    "args": {
      "External id": 2469019,"Record function id": 0, "Ev Idx": 2714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194065.855, "dur": 2.635,
    "args": {
      "External id": 2469020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194066.419, "dur": 1.633,
    "args": {
      "External id": 2469021,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194066.945, "dur": 1.026,
    "args": {
      "External id": 2469022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194072.357, "dur": 6.726,
    "args": {
      "External id": 2469023,"Record function id": 0, "Ev Idx": 2718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194073.937, "dur": 4.741,
    "args": {
      "External id": 2469024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194074.425, "dur": 3.693,
    "args": {
      "External id": 2469025,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194075.280, "dur": 2.777,
    "args": {
      "External id": 2469026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194082.767, "dur": 4.540,
    "args": {
      "External id": 2469027,"Record function id": 0, "Ev Idx": 2722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194084.132, "dur": 2.747,
    "args": {
      "External id": 2469028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194084.972, "dur": 1.278,
    "args": {
      "External id": 2469029,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194085.461, "dur": 0.723,
    "args": {
      "External id": 2469030,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194090.773, "dur": 4.177,
    "args": {
      "External id": 2469031,"Record function id": 0, "Ev Idx": 2726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194092.227, "dur": 2.323,
    "args": {
      "External id": 2469032,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194092.901, "dur": 1.215,
    "args": {
      "External id": 2469033,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194093.425, "dur": 0.581,
    "args": {
      "External id": 2469034,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194098.314, "dur": 3.650,
    "args": {
      "External id": 2469035,"Record function id": 0, "Ev Idx": 2730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194099.403, "dur": 2.180,
    "args": {
      "External id": 2469036,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194099.872, "dur": 1.158,
    "args": {
      "External id": 2469037,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194100.378, "dur": 0.565,
    "args": {
      "External id": 2469038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194105.260, "dur": 3.945,
    "args": {
      "External id": 2469039,"Record function id": 0, "Ev Idx": 2734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194106.469, "dur": 2.364,
    "args": {
      "External id": 2469040,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194106.958, "dur": 1.276,
    "args": {
      "External id": 2469041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194107.357, "dur": 0.807,
    "args": {
      "External id": 2469042,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194112.487, "dur": 3.878,
    "args": {
      "External id": 2469043,"Record function id": 0, "Ev Idx": 2738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194113.578, "dur": 2.407,
    "args": {
      "External id": 2469044,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194114.052, "dur": 1.414,
    "args": {
      "External id": 2469045,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194114.596, "dur": 0.780,
    "args": {
      "External id": 2469046,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194120.405, "dur": 36775.947,
    "args": {
      "External id": 2469047,"Record function id": 0, "Sequence number": 24740461, "Fwd thread id": 1, "Ev Idx": 2742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194121.798, "dur": 36766.379,
    "args": {
      "External id": 2469048,"Sequence number": 24740461, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2743
    }
  },
  {
    "ph": "f", "id": 187, "pid": 1336756, "tid": 1381189, "ts": 1514657194121.798,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.20)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194151.839, "dur": 42.319,
    "args": {
      "External id": 2469049,"Record function id": 0, "Ev Idx": 2744
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.20)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194201.742, "dur": 71.463,
    "args": {
      "External id": 2469050,"Record function id": 0, "Ev Idx": 2745
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.20)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194278.955, "dur": 36602.034,
    "args": {
      "External id": 2469051,"Record function id": 0, "Ev Idx": 2746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194372.992, "dur": 6.507,
    "args": {
      "External id": 2469052,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194389.214, "dur": 4.585,
    "args": {
      "External id": 2469053,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194408.884, "dur": 35643.423,
    "args": {
      "External id": 2469054,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194422.350, "dur": 35619.937,
    "args": {
      "External id": 2469055,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194507.216, "dur": 15.479,
    "args": {
      "External id": 2469056,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194545.105, "dur": 35430.981,
    "args": {
      "External id": 2469057,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194547.881, "dur": 35427.550,
    "args": {
      "External id": 2469058,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194552.693, "dur": 6.554,
    "args": {
      "External id": 2469059,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657194561.309, "dur": 35410.650,
    "args": {
      "External id": 2469060,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230141.794, "dur": 8.501,
    "args": {
      "External id": 2469061,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230144.865, "dur": 5.082,
    "args": {
      "External id": 2469062,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230185.197, "dur": 387.355,
    "args": {
      "External id": 2469063,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230215.804, "dur": 351.400,
    "args": {
      "External id": 2469064,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2759, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230226.328, "dur": 335.033,
    "args": {
      "External id": 2469065,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230594.288, "dur": 2.431,
    "args": {
      "External id": 2469066,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2761, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230658.140, "dur": 6.653,
    "args": {
      "External id": 2469067,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230712.046, "dur": 1.498,
    "args": {
      "External id": 2469068,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230731.546, "dur": 0.906,
    "args": {
      "External id": 2469069,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230748.104, "dur": 0.902,
    "args": {
      "External id": 2469070,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230761.631, "dur": 1.101,
    "args": {
      "External id": 2469071,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230774.825, "dur": 1.069,
    "args": {
      "External id": 2469072,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230788.577, "dur": 0.920,
    "args": {
      "External id": 2469073,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230804.293, "dur": 2.269,
    "args": {
      "External id": 2469074,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230818.889, "dur": 0.965,
    "args": {
      "External id": 2469075,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230911.391, "dur": 2823.342,
    "args": {
      "External id": 2469076,"Record function id": 0, "Ev Idx": 2771
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.19)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230931.065, "dur": 1090.707,
    "args": {
      "External id": 2469077,"Record function id": 0, "Ev Idx": 2772
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657230945.529, "dur": 369.971,
    "args": {
      "External id": 2469078,"Record function id": 0, "Ev Idx": 2773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231070.704, "dur": 4.757,
    "args": {
      "External id": 2469079,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231078.782, "dur": 1.538,
    "args": {
      "External id": 2469080,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231082.203, "dur": 1.066,
    "args": {
      "External id": 2469081,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231085.212, "dur": 2.581,
    "args": {
      "External id": 2469082,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231089.580, "dur": 1.080,
    "args": {
      "External id": 2469083,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231098.237, "dur": 1.131,
    "args": {
      "External id": 2469084,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231101.188, "dur": 2.093,
    "args": {
      "External id": 2469085,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231104.891, "dur": 0.906,
    "args": {
      "External id": 2469086,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231107.597, "dur": 0.914,
    "args": {
      "External id": 2469087,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231109.995, "dur": 0.946,
    "args": {
      "External id": 2469088,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231130.344, "dur": 150.885,
    "args": {
      "External id": 2469089,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231145.923, "dur": 130.833,
    "args": {
      "External id": 2469090,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231163.115, "dur": 12.132,
    "args": {
      "External id": 2469091,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231178.539, "dur": 68.667,
    "args": {
      "External id": 2469092,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231181.513, "dur": 65.389,
    "args": {
      "External id": 2469093,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231186.061, "dur": 6.961,
    "args": {
      "External id": 2469094,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231195.301, "dur": 51.097,
    "args": {
      "External id": 2469095,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2790
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.18", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231407.326, "dur": 566.712,
    "args": {
      "External id": 2469096,"Record function id": 0, "Ev Idx": 2791
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231427.055, "dur": 534.573,
    "args": {
      "External id": 2469097,"Record function id": 0, "Ev Idx": 2792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231490.981, "dur": 4.288,
    "args": {
      "External id": 2469098,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231510.881, "dur": 48.540,
    "args": {
      "External id": 2469099,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231515.810, "dur": 1.384,
    "args": {
      "External id": 2469100,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231519.506, "dur": 0.463,
    "args": {
      "External id": 2469101,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231522.127, "dur": 0.488,
    "args": {
      "External id": 2469102,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231524.866, "dur": 0.373,
    "args": {
      "External id": 2469103,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231527.179, "dur": 16.452,
    "args": {
      "External id": 2469104,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231546.609, "dur": 0.668,
    "args": {
      "External id": 2469105,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231548.833, "dur": 2.061,
    "args": {
      "External id": 2469106,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231552.677, "dur": 0.395,
    "args": {
      "External id": 2469107,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231554.844, "dur": 0.324,
    "args": {
      "External id": 2469108,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231571.122, "dur": 34.688,
    "args": {
      "External id": 2469109,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231639.365, "dur": 102.539,
    "args": {
      "External id": 2469110,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231650.436, "dur": 4.214,
    "args": {
      "External id": 2469111,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231660.210, "dur": 10.363,
    "args": {
      "External id": 2469112,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231664.481, "dur": 5.672,
    "args": {
      "External id": 2469113,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231668.467, "dur": 0.609,
    "args": {
      "External id": 2469114,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231677.746, "dur": 26.352,
    "args": {
      "External id": 2469115,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231680.476, "dur": 0.594,
    "args": {
      "External id": 2469116,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231683.072, "dur": 0.494,
    "args": {
      "External id": 2469117,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231685.316, "dur": 0.422,
    "args": {
      "External id": 2469118,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231687.689, "dur": 2.157,
    "args": {
      "External id": 2469119,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231691.447, "dur": 0.441,
    "args": {
      "External id": 2469120,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231694.266, "dur": 0.352,
    "args": {
      "External id": 2469121,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231696.299, "dur": 0.426,
    "args": {
      "External id": 2469122,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231698.856, "dur": 0.450,
    "args": {
      "External id": 2469123,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231700.837, "dur": 0.292,
    "args": {
      "External id": 2469124,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231713.136, "dur": 21.835,
    "args": {
      "External id": 2469125,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231789.899, "dur": 109.538,
    "args": {
      "External id": 2469126,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231812.515, "dur": 83.091,
    "args": {
      "External id": 2469127,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2822, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231822.033, "dur": 69.444,
    "args": {
      "External id": 2469128,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657231913.441, "dur": 1.702,
    "args": {
      "External id": 2469129,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2824, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232032.550, "dur": 1679.989,
    "args": {
      "External id": 2469130,"Sequence number": 24740460, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2825
    }
  },
  {
    "ph": "f", "id": 188, "pid": 1336756, "tid": 1381189, "ts": 1514657232032.550,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232147.282, "dur": 107.966,
    "args": {
      "External id": 2469131,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232294.788, "dur": 42.708,
    "args": {
      "External id": 2469132,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232356.072, "dur": 50.244,
    "args": {
      "External id": 2469133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232416.628, "dur": 33.840,
    "args": {
      "External id": 2469134,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232460.180, "dur": 47.466,
    "args": {
      "External id": 2469135,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232516.479, "dur": 46.257,
    "args": {
      "External id": 2469136,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232574.562, "dur": 47.773,
    "args": {
      "External id": 2469137,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232645.457, "dur": 25.313,
    "args": {
      "External id": 2469138,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232691.280, "dur": 30.769,
    "args": {
      "External id": 2469139,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232741.616, "dur": 20.624,
    "args": {
      "External id": 2469140,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232774.809, "dur": 16.208,
    "args": {
      "External id": 2469141,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232800.385, "dur": 29.838,
    "args": {
      "External id": 2469142,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232833.338, "dur": 33.391,
    "args": {
      "External id": 2469143,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232895.227, "dur": 222.459,
    "args": {
      "External id": 2469144,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657232975.215, "dur": 43.183,
    "args": {
      "External id": 2469145,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233022.059, "dur": 3.135,
    "args": {
      "External id": 2469146,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233152.376, "dur": 24.952,
    "args": {
      "External id": 2469147,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233189.045, "dur": 16.843,
    "args": {
      "External id": 2469148,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233215.198, "dur": 41.441,
    "args": {
      "External id": 2469149,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233263.378, "dur": 36.325,
    "args": {
      "External id": 2469150,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233308.852, "dur": 22.822,
    "args": {
      "External id": 2469151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233337.190, "dur": 35.725,
    "args": {
      "External id": 2469152,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233394.134, "dur": 34.052,
    "args": {
      "External id": 2469153,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233436.596, "dur": 33.394,
    "args": {
      "External id": 2469154,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233487.887, "dur": 27.616,
    "args": {
      "External id": 2469155,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233546.524, "dur": 30.801,
    "args": {
      "External id": 2469156,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233594.525, "dur": 19.482,
    "args": {
      "External id": 2469157,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233634.168, "dur": 14.981,
    "args": {
      "External id": 2469158,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233662.288, "dur": 22.147,
    "args": {
      "External id": 2469159,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233756.566, "dur": 16.294,
    "args": {
      "External id": 2469160,"Record function id": 0, "Ev Idx": 2855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233760.549, "dur": 11.310,
    "args": {
      "External id": 2469161,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233764.712, "dur": 6.215,
    "args": {
      "External id": 2469162,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233766.677, "dur": 4.127,
    "args": {
      "External id": 2469163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233776.714, "dur": 5.430,
    "args": {
      "External id": 2469164,"Record function id": 0, "Ev Idx": 2859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233778.327, "dur": 3.396,
    "args": {
      "External id": 2469165,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233779.147, "dur": 2.009,
    "args": {
      "External id": 2469166,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233780.102, "dur": 0.948,
    "args": {
      "External id": 2469167,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233785.382, "dur": 6.467,
    "args": {
      "External id": 2469168,"Record function id": 0, "Ev Idx": 2863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233786.850, "dur": 4.601,
    "args": {
      "External id": 2469169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233787.664, "dur": 3.324,
    "args": {
      "External id": 2469170,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233788.236, "dur": 2.639,
    "args": {
      "External id": 2469171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233795.033, "dur": 5.032,
    "args": {
      "External id": 2469172,"Record function id": 0, "Ev Idx": 2867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233796.631, "dur": 3.043,
    "args": {
      "External id": 2469173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233797.555, "dur": 1.441,
    "args": {
      "External id": 2469174,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233797.867, "dur": 1.061,
    "args": {
      "External id": 2469175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233803.271, "dur": 3.813,
    "args": {
      "External id": 2469176,"Record function id": 0, "Ev Idx": 2871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233804.504, "dur": 2.187,
    "args": {
      "External id": 2469177,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233805.055, "dur": 0.950,
    "args": {
      "External id": 2469178,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233805.362, "dur": 0.570,
    "args": {
      "External id": 2469179,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233810.268, "dur": 3.927,
    "args": {
      "External id": 2469180,"Record function id": 0, "Ev Idx": 2875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233811.384, "dur": 2.394,
    "args": {
      "External id": 2469181,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233812.039, "dur": 1.218,
    "args": {
      "External id": 2469182,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233812.484, "dur": 0.650,
    "args": {
      "External id": 2469183,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233817.567, "dur": 3.888,
    "args": {
      "External id": 2469184,"Record function id": 0, "Ev Idx": 2879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233818.985, "dur": 2.079,
    "args": {
      "External id": 2469185,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233819.518, "dur": 1.151,
    "args": {
      "External id": 2469186,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233819.940, "dur": 0.641,
    "args": {
      "External id": 2469187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233824.714, "dur": 3.574,
    "args": {
      "External id": 2469188,"Record function id": 0, "Ev Idx": 2883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233825.694, "dur": 2.203,
    "args": {
      "External id": 2469189,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233826.160, "dur": 1.104,
    "args": {
      "External id": 2469190,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233826.449, "dur": 0.751,
    "args": {
      "External id": 2469191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233831.316, "dur": 4.164,
    "args": {
      "External id": 2469192,"Record function id": 0, "Ev Idx": 2887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233832.669, "dur": 2.420,
    "args": {
      "External id": 2469193,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233833.300, "dur": 1.114,
    "args": {
      "External id": 2469194,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233833.709, "dur": 0.617,
    "args": {
      "External id": 2469195,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233839.575, "dur": 37315.949,
    "args": {
      "External id": 2469196,"Record function id": 0, "Sequence number": 24740459, "Fwd thread id": 1, "Ev Idx": 2891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233841.190, "dur": 37305.709,
    "args": {
      "External id": 2469197,"Sequence number": 24740459, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2892
    }
  },
  {
    "ph": "f", "id": 189, "pid": 1336756, "tid": 1381189, "ts": 1514657233841.190,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.19)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233869.336, "dur": 36.657,
    "args": {
      "External id": 2469198,"Record function id": 0, "Ev Idx": 2893
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.19)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657233913.249, "dur": 64.313,
    "args": {
      "External id": 2469199,"Record function id": 0, "Ev Idx": 2894
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.19)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234018.625, "dur": 37120.081,
    "args": {
      "External id": 2469200,"Record function id": 0, "Ev Idx": 2895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234110.675, "dur": 6.926,
    "args": {
      "External id": 2469201,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234128.622, "dur": 6.580,
    "args": {
      "External id": 2469202,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234150.399, "dur": 36207.090,
    "args": {
      "External id": 2469203,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234166.995, "dur": 36181.322,
    "args": {
      "External id": 2469204,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234246.471, "dur": 13.729,
    "args": {
      "External id": 2469205,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234266.835, "dur": 36040.898,
    "args": {
      "External id": 2469206,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234269.596, "dur": 36037.504,
    "args": {
      "External id": 2469207,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234273.871, "dur": 4.855,
    "args": {
      "External id": 2469208,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657234281.145, "dur": 36022.365,
    "args": {
      "External id": 2469209,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270448.908, "dur": 8.659,
    "args": {
      "External id": 2469210,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270452.077, "dur": 5.188,
    "args": {
      "External id": 2469211,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270489.186, "dur": 301.635,
    "args": {
      "External id": 2469212,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270516.087, "dur": 270.003,
    "args": {
      "External id": 2469213,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2908, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270537.940, "dur": 242.570,
    "args": {
      "External id": 2469214,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270812.238, "dur": 2.005,
    "args": {
      "External id": 2469215,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2910, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270871.760, "dur": 6.219,
    "args": {
      "External id": 2469216,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270926.393, "dur": 1.437,
    "args": {
      "External id": 2469217,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270945.620, "dur": 2.813,
    "args": {
      "External id": 2469218,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270961.840, "dur": 1.036,
    "args": {
      "External id": 2469219,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657270975.845, "dur": 0.917,
    "args": {
      "External id": 2469220,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271030.600, "dur": 1.703,
    "args": {
      "External id": 2469221,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271046.066, "dur": 2.581,
    "args": {
      "External id": 2469222,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271061.284, "dur": 1.957,
    "args": {
      "External id": 2469223,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271075.268, "dur": 1.007,
    "args": {
      "External id": 2469224,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271171.999, "dur": 2859.109,
    "args": {
      "External id": 2469225,"Record function id": 0, "Ev Idx": 2920
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.18)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271191.924, "dur": 1072.623,
    "args": {
      "External id": 2469226,"Record function id": 0, "Ev Idx": 2921
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271207.374, "dur": 341.746,
    "args": {
      "External id": 2469227,"Record function id": 0, "Ev Idx": 2922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271292.334, "dur": 3.925,
    "args": {
      "External id": 2469228,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271299.663, "dur": 1.179,
    "args": {
      "External id": 2469229,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271302.839, "dur": 3.013,
    "args": {
      "External id": 2469230,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271307.474, "dur": 0.884,
    "args": {
      "External id": 2469231,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271310.004, "dur": 0.785,
    "args": {
      "External id": 2469232,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271312.907, "dur": 1.036,
    "args": {
      "External id": 2469233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271315.567, "dur": 1.833,
    "args": {
      "External id": 2469234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271319.049, "dur": 0.916,
    "args": {
      "External id": 2469235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271321.534, "dur": 0.853,
    "args": {
      "External id": 2469236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271324.218, "dur": 0.844,
    "args": {
      "External id": 2469237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271343.450, "dur": 158.069,
    "args": {
      "External id": 2469238,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271359.217, "dur": 138.319,
    "args": {
      "External id": 2469239,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271383.908, "dur": 13.728,
    "args": {
      "External id": 2469240,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271401.114, "dur": 66.634,
    "args": {
      "External id": 2469241,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271404.051, "dur": 63.334,
    "args": {
      "External id": 2469242,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271408.209, "dur": 5.024,
    "args": {
      "External id": 2469243,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271415.379, "dur": 51.285,
    "args": {
      "External id": 2469244,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2939
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.17", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271646.078, "dur": 609.606,
    "args": {
      "External id": 2469245,"Record function id": 0, "Ev Idx": 2940
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271667.451, "dur": 575.312,
    "args": {
      "External id": 2469246,"Record function id": 0, "Ev Idx": 2941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271728.636, "dur": 5.097,
    "args": {
      "External id": 2469247,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271749.463, "dur": 35.933,
    "args": {
      "External id": 2469248,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271754.626, "dur": 1.415,
    "args": {
      "External id": 2469249,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271758.048, "dur": 0.737,
    "args": {
      "External id": 2469250,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271761.008, "dur": 0.469,
    "args": {
      "External id": 2469251,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271763.242, "dur": 0.338,
    "args": {
      "External id": 2469252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271765.287, "dur": 0.570,
    "args": {
      "External id": 2469253,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271767.424, "dur": 2.223,
    "args": {
      "External id": 2469254,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271776.105, "dur": 0.468,
    "args": {
      "External id": 2469255,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271778.471, "dur": 0.376,
    "args": {
      "External id": 2469256,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271780.592, "dur": 0.381,
    "args": {
      "External id": 2469257,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271796.158, "dur": 35.437,
    "args": {
      "External id": 2469258,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271864.846, "dur": 103.646,
    "args": {
      "External id": 2469259,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271874.713, "dur": 3.370,
    "args": {
      "External id": 2469260,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271883.501, "dur": 10.126,
    "args": {
      "External id": 2469261,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271887.540, "dur": 5.674,
    "args": {
      "External id": 2469262,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271891.375, "dur": 0.617,
    "args": {
      "External id": 2469263,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271900.212, "dur": 25.677,
    "args": {
      "External id": 2469264,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271902.424, "dur": 0.477,
    "args": {
      "External id": 2469265,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271904.796, "dur": 0.663,
    "args": {
      "External id": 2469266,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271907.386, "dur": 2.302,
    "args": {
      "External id": 2469267,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271911.544, "dur": 0.352,
    "args": {
      "External id": 2469268,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271913.618, "dur": 0.396,
    "args": {
      "External id": 2469269,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271915.683, "dur": 0.356,
    "args": {
      "External id": 2469270,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271917.899, "dur": 0.428,
    "args": {
      "External id": 2469271,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271920.007, "dur": 0.442,
    "args": {
      "External id": 2469272,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271922.158, "dur": 0.512,
    "args": {
      "External id": 2469273,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657271940.120, "dur": 21.043,
    "args": {
      "External id": 2469274,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272056.505, "dur": 116.940,
    "args": {
      "External id": 2469275,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272082.064, "dur": 87.641,
    "args": {
      "External id": 2469276,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2971, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272092.294, "dur": 72.957,
    "args": {
      "External id": 2469277,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272189.801, "dur": 1.875,
    "args": {
      "External id": 2469278,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2973, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272272.858, "dur": 1691.572,
    "args": {
      "External id": 2469279,"Sequence number": 24740458, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2974
    }
  },
  {
    "ph": "f", "id": 190, "pid": 1336756, "tid": 1381189, "ts": 1514657272272.858,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272388.089, "dur": 104.662,
    "args": {
      "External id": 2469280,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272544.828, "dur": 46.346,
    "args": {
      "External id": 2469281,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272611.918, "dur": 58.204,
    "args": {
      "External id": 2469282,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272682.246, "dur": 35.386,
    "args": {
      "External id": 2469283,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272725.031, "dur": 49.338,
    "args": {
      "External id": 2469284,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272782.629, "dur": 28.444,
    "args": {
      "External id": 2469285,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272819.539, "dur": 43.025,
    "args": {
      "External id": 2469286,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272888.139, "dur": 25.549,
    "args": {
      "External id": 2469287,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657272932.063, "dur": 30.628,
    "args": {
      "External id": 2469288,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273018.951, "dur": 23.906,
    "args": {
      "External id": 2469289,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273057.969, "dur": 17.069,
    "args": {
      "External id": 2469290,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273086.351, "dur": 36.809,
    "args": {
      "External id": 2469291,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273127.402, "dur": 37.470,
    "args": {
      "External id": 2469292,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273195.702, "dur": 179.703,
    "args": {
      "External id": 2469293,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273277.765, "dur": 6.214,
    "args": {
      "External id": 2469294,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273286.197, "dur": 2.142,
    "args": {
      "External id": 2469295,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273409.975, "dur": 23.112,
    "args": {
      "External id": 2469296,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273444.981, "dur": 17.072,
    "args": {
      "External id": 2469297,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273470.962, "dur": 35.364,
    "args": {
      "External id": 2469298,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273513.355, "dur": 56.004,
    "args": {
      "External id": 2469299,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273597.135, "dur": 36.321,
    "args": {
      "External id": 2469300,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273641.901, "dur": 34.171,
    "args": {
      "External id": 2469301,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273684.140, "dur": 20.825,
    "args": {
      "External id": 2469302,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273713.880, "dur": 31.199,
    "args": {
      "External id": 2469303,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273769.935, "dur": 25.619,
    "args": {
      "External id": 2469304,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273816.953, "dur": 24.733,
    "args": {
      "External id": 2469305,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273857.444, "dur": 19.805,
    "args": {
      "External id": 2469306,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273891.542, "dur": 14.399,
    "args": {
      "External id": 2469307,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657273918.082, "dur": 18.339,
    "args": {
      "External id": 2469308,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274056.388, "dur": 16.539,
    "args": {
      "External id": 2469309,"Record function id": 0, "Ev Idx": 3004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274059.905, "dur": 11.873,
    "args": {
      "External id": 2469310,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274064.278, "dur": 6.100,
    "args": {
      "External id": 2469311,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274065.898, "dur": 4.359,
    "args": {
      "External id": 2469312,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274076.881, "dur": 7.488,
    "args": {
      "External id": 2469313,"Record function id": 0, "Ev Idx": 3008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274078.292, "dur": 5.655,
    "args": {
      "External id": 2469314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274079.080, "dur": 4.293,
    "args": {
      "External id": 2469315,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274079.836, "dur": 3.410,
    "args": {
      "External id": 2469316,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274087.673, "dur": 4.373,
    "args": {
      "External id": 2469317,"Record function id": 0, "Ev Idx": 3012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274089.025, "dur": 2.608,
    "args": {
      "External id": 2469318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274089.512, "dur": 1.672,
    "args": {
      "External id": 2469319,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274090.143, "dur": 0.930,
    "args": {
      "External id": 2469320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274095.263, "dur": 4.030,
    "args": {
      "External id": 2469321,"Record function id": 0, "Ev Idx": 3016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274096.567, "dur": 2.342,
    "args": {
      "External id": 2469322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274097.293, "dur": 1.179,
    "args": {
      "External id": 2469323,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274097.666, "dur": 0.690,
    "args": {
      "External id": 2469324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274102.342, "dur": 4.227,
    "args": {
      "External id": 2469325,"Record function id": 0, "Ev Idx": 3020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274103.699, "dur": 2.461,
    "args": {
      "External id": 2469326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274104.602, "dur": 1.183,
    "args": {
      "External id": 2469327,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274105.112, "dur": 0.601,
    "args": {
      "External id": 2469328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274109.639, "dur": 4.530,
    "args": {
      "External id": 2469329,"Record function id": 0, "Ev Idx": 3024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274110.772, "dur": 2.988,
    "args": {
      "External id": 2469330,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274111.625, "dur": 1.540,
    "args": {
      "External id": 2469331,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274112.422, "dur": 0.620,
    "args": {
      "External id": 2469332,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274117.560, "dur": 3.738,
    "args": {
      "External id": 2469333,"Record function id": 0, "Ev Idx": 3028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274118.651, "dur": 2.241,
    "args": {
      "External id": 2469334,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274119.101, "dur": 1.182,
    "args": {
      "External id": 2469335,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274119.558, "dur": 0.643,
    "args": {
      "External id": 2469336,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274124.375, "dur": 3.836,
    "args": {
      "External id": 2469337,"Record function id": 0, "Ev Idx": 3032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274125.351, "dur": 2.448,
    "args": {
      "External id": 2469338,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274126.129, "dur": 1.117,
    "args": {
      "External id": 2469339,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274126.434, "dur": 0.739,
    "args": {
      "External id": 2469340,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274131.225, "dur": 5.813,
    "args": {
      "External id": 2469341,"Record function id": 0, "Ev Idx": 3036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274132.304, "dur": 4.339,
    "args": {
      "External id": 2469342,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274132.840, "dur": 3.259,
    "args": {
      "External id": 2469343,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274133.460, "dur": 2.538,
    "args": {
      "External id": 2469344,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274140.701, "dur": 36477.976,
    "args": {
      "External id": 2469345,"Record function id": 0, "Sequence number": 24740457, "Fwd thread id": 1, "Ev Idx": 3040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274142.158, "dur": 36467.136,
    "args": {
      "External id": 2469346,"Sequence number": 24740457, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3041
    }
  },
  {
    "ph": "f", "id": 191, "pid": 1336756, "tid": 1381189, "ts": 1514657274142.158,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.18)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274172.542, "dur": 38.312,
    "args": {
      "External id": 2469347,"Record function id": 0, "Ev Idx": 3042
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.18)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274218.720, "dur": 65.621,
    "args": {
      "External id": 2469348,"Record function id": 0, "Ev Idx": 3043
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.18)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274289.826, "dur": 36311.240,
    "args": {
      "External id": 2469349,"Record function id": 0, "Ev Idx": 3044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274377.445, "dur": 6.582,
    "args": {
      "External id": 2469350,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274394.004, "dur": 4.940,
    "args": {
      "External id": 2469351,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274414.460, "dur": 35310.123,
    "args": {
      "External id": 2469352,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274463.750, "dur": 35252.024,
    "args": {
      "External id": 2469353,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274508.032, "dur": 13.590,
    "args": {
      "External id": 2469354,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274545.251, "dur": 35129.682,
    "args": {
      "External id": 2469355,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274548.366, "dur": 35126.014,
    "args": {
      "External id": 2469356,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274552.837, "dur": 10.393,
    "args": {
      "External id": 2469357,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657274565.366, "dur": 35105.722,
    "args": {
      "External id": 2469358,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657309811.483, "dur": 8.626,
    "args": {
      "External id": 2469359,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657309814.845, "dur": 4.886,
    "args": {
      "External id": 2469360,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657309850.972, "dur": 422.219,
    "args": {
      "External id": 2469361,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657309877.907, "dur": 388.963,
    "args": {
      "External id": 2469362,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3057, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657309888.972, "dur": 372.070,
    "args": {
      "External id": 2469363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310295.801, "dur": 2.529,
    "args": {
      "External id": 2469364,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3059, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310361.879, "dur": 6.653,
    "args": {
      "External id": 2469365,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310415.986, "dur": 3.032,
    "args": {
      "External id": 2469366,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310437.390, "dur": 1.000,
    "args": {
      "External id": 2469367,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310451.114, "dur": 0.938,
    "args": {
      "External id": 2469368,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310463.819, "dur": 1.289,
    "args": {
      "External id": 2469369,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310477.263, "dur": 2.889,
    "args": {
      "External id": 2469370,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310492.240, "dur": 0.845,
    "args": {
      "External id": 2469371,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310505.672, "dur": 1.786,
    "args": {
      "External id": 2469372,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310519.180, "dur": 0.792,
    "args": {
      "External id": 2469373,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310636.167, "dur": 2852.478,
    "args": {
      "External id": 2469374,"Record function id": 0, "Ev Idx": 3069
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.17)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310656.293, "dur": 1107.677,
    "args": {
      "External id": 2469375,"Record function id": 0, "Ev Idx": 3070
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310670.252, "dur": 394.565,
    "args": {
      "External id": 2469376,"Record function id": 0, "Ev Idx": 3071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310786.744, "dur": 6.249,
    "args": {
      "External id": 2469377,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310796.663, "dur": 1.075,
    "args": {
      "External id": 2469378,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310799.525, "dur": 0.818,
    "args": {
      "External id": 2469379,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310802.005, "dur": 0.956,
    "args": {
      "External id": 2469380,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310804.615, "dur": 0.682,
    "args": {
      "External id": 2469381,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310808.847, "dur": 0.824,
    "args": {
      "External id": 2469382,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310811.694, "dur": 1.516,
    "args": {
      "External id": 2469383,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310814.707, "dur": 1.027,
    "args": {
      "External id": 2469384,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310817.556, "dur": 3.046,
    "args": {
      "External id": 2469385,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310822.095, "dur": 0.767,
    "args": {
      "External id": 2469386,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310841.642, "dur": 185.265,
    "args": {
      "External id": 2469387,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310858.365, "dur": 163.090,
    "args": {
      "External id": 2469388,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310873.389, "dur": 12.413,
    "args": {
      "External id": 2469389,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310889.023, "dur": 65.981,
    "args": {
      "External id": 2469390,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310891.409, "dur": 63.317,
    "args": {
      "External id": 2469391,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310895.761, "dur": 5.179,
    "args": {
      "External id": 2469392,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657310902.727, "dur": 51.275,
    "args": {
      "External id": 2469393,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3088
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.16", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311167.228, "dur": 588.622,
    "args": {
      "External id": 2469394,"Record function id": 0, "Ev Idx": 3089
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311186.966, "dur": 553.308,
    "args": {
      "External id": 2469395,"Record function id": 0, "Ev Idx": 3090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311246.403, "dur": 5.066,
    "args": {
      "External id": 2469396,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311266.891, "dur": 31.663,
    "args": {
      "External id": 2469397,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311271.774, "dur": 1.742,
    "args": {
      "External id": 2469398,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311275.972, "dur": 0.417,
    "args": {
      "External id": 2469399,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311277.976, "dur": 0.653,
    "args": {
      "External id": 2469400,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311280.242, "dur": 2.830,
    "args": {
      "External id": 2469401,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311284.876, "dur": 0.512,
    "args": {
      "External id": 2469402,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311287.011, "dur": 0.397,
    "args": {
      "External id": 2469403,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311289.304, "dur": 0.394,
    "args": {
      "External id": 2469404,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311291.436, "dur": 0.614,
    "args": {
      "External id": 2469405,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311293.927, "dur": 0.448,
    "args": {
      "External id": 2469406,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311308.750, "dur": 33.241,
    "args": {
      "External id": 2469407,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311375.353, "dur": 109.035,
    "args": {
      "External id": 2469408,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311385.308, "dur": 2.687,
    "args": {
      "External id": 2469409,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311393.646, "dur": 10.567,
    "args": {
      "External id": 2469410,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311397.913, "dur": 5.869,
    "args": {
      "External id": 2469411,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311402.098, "dur": 0.642,
    "args": {
      "External id": 2469412,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311410.887, "dur": 27.643,
    "args": {
      "External id": 2469413,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311413.103, "dur": 2.123,
    "args": {
      "External id": 2469414,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311417.101, "dur": 0.463,
    "args": {
      "External id": 2469415,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311419.704, "dur": 0.463,
    "args": {
      "External id": 2469416,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311421.735, "dur": 0.370,
    "args": {
      "External id": 2469417,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311424.394, "dur": 0.375,
    "args": {
      "External id": 2469418,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311426.705, "dur": 0.445,
    "args": {
      "External id": 2469419,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311428.941, "dur": 0.427,
    "args": {
      "External id": 2469420,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311431.084, "dur": 0.286,
    "args": {
      "External id": 2469421,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311433.150, "dur": 2.073,
    "args": {
      "External id": 2469422,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311455.407, "dur": 21.865,
    "args": {
      "External id": 2469423,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311546.215, "dur": 120.314,
    "args": {
      "External id": 2469424,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311573.431, "dur": 89.456,
    "args": {
      "External id": 2469425,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3120, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311584.753, "dur": 73.963,
    "args": {
      "External id": 2469426,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311683.855, "dur": 1.569,
    "args": {
      "External id": 2469427,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3122, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311771.325, "dur": 1696.078,
    "args": {
      "External id": 2469428,"Sequence number": 24740456, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3123
    }
  },
  {
    "ph": "f", "id": 192, "pid": 1336756, "tid": 1381189, "ts": 1514657311771.325,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657311884.550, "dur": 142.697,
    "args": {
      "External id": 2469429,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312068.069, "dur": 43.803,
    "args": {
      "External id": 2469430,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312133.850, "dur": 58.112,
    "args": {
      "External id": 2469431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312202.024, "dur": 34.676,
    "args": {
      "External id": 2469432,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312244.590, "dur": 46.997,
    "args": {
      "External id": 2469433,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312299.749, "dur": 28.721,
    "args": {
      "External id": 2469434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312338.991, "dur": 43.231,
    "args": {
      "External id": 2469435,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312404.319, "dur": 22.990,
    "args": {
      "External id": 2469436,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312444.443, "dur": 30.653,
    "args": {
      "External id": 2469437,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312493.239, "dur": 19.351,
    "args": {
      "External id": 2469438,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312524.772, "dur": 32.241,
    "args": {
      "External id": 2469439,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312570.425, "dur": 33.953,
    "args": {
      "External id": 2469440,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312608.145, "dur": 34.071,
    "args": {
      "External id": 2469441,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312670.058, "dur": 173.718,
    "args": {
      "External id": 2469442,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312747.506, "dur": 5.698,
    "args": {
      "External id": 2469443,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312755.553, "dur": 2.123,
    "args": {
      "External id": 2469444,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312878.063, "dur": 24.618,
    "args": {
      "External id": 2469445,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312929.824, "dur": 18.028,
    "args": {
      "External id": 2469446,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657312956.820, "dur": 69.781,
    "args": {
      "External id": 2469447,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313038.802, "dur": 56.244,
    "args": {
      "External id": 2469448,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313110.062, "dur": 28.316,
    "args": {
      "External id": 2469449,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313144.005, "dur": 34.110,
    "args": {
      "External id": 2469450,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313185.224, "dur": 25.375,
    "args": {
      "External id": 2469451,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313221.076, "dur": 34.925,
    "args": {
      "External id": 2469452,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313276.143, "dur": 25.697,
    "args": {
      "External id": 2469453,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313319.689, "dur": 24.629,
    "args": {
      "External id": 2469454,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313359.766, "dur": 19.385,
    "args": {
      "External id": 2469455,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313393.604, "dur": 15.680,
    "args": {
      "External id": 2469456,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313421.648, "dur": 17.864,
    "args": {
      "External id": 2469457,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313511.377, "dur": 35.904,
    "args": {
      "External id": 2469458,"Record function id": 0, "Ev Idx": 3153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313514.673, "dur": 12.408,
    "args": {
      "External id": 2469459,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313518.759, "dur": 7.561,
    "args": {
      "External id": 2469460,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313520.634, "dur": 5.569,
    "args": {
      "External id": 2469461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313554.266, "dur": 7.668,
    "args": {
      "External id": 2469462,"Record function id": 0, "Ev Idx": 3157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313556.293, "dur": 5.161,
    "args": {
      "External id": 2469463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313557.856, "dur": 2.793,
    "args": {
      "External id": 2469464,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313558.790, "dur": 1.739,
    "args": {
      "External id": 2469465,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313565.194, "dur": 4.636,
    "args": {
      "External id": 2469466,"Record function id": 0, "Ev Idx": 3161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313566.739, "dur": 2.690,
    "args": {
      "External id": 2469467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313567.454, "dur": 1.543,
    "args": {
      "External id": 2469468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313567.799, "dur": 1.113,
    "args": {
      "External id": 2469469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313573.005, "dur": 4.492,
    "args": {
      "External id": 2469470,"Record function id": 0, "Ev Idx": 3165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313574.485, "dur": 2.392,
    "args": {
      "External id": 2469471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313575.017, "dur": 1.099,
    "args": {
      "External id": 2469472,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313575.333, "dur": 0.707,
    "args": {
      "External id": 2469473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313580.657, "dur": 4.092,
    "args": {
      "External id": 2469474,"Record function id": 0, "Ev Idx": 3169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313582.139, "dur": 2.214,
    "args": {
      "External id": 2469475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313582.621, "dur": 1.319,
    "args": {
      "External id": 2469476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313583.272, "dur": 0.596,
    "args": {
      "External id": 2469477,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313588.089, "dur": 3.809,
    "args": {
      "External id": 2469478,"Record function id": 0, "Ev Idx": 3173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313589.452, "dur": 2.024,
    "args": {
      "External id": 2469479,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313589.955, "dur": 1.105,
    "args": {
      "External id": 2469480,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313590.380, "dur": 0.572,
    "args": {
      "External id": 2469481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313595.096, "dur": 6.650,
    "args": {
      "External id": 2469482,"Record function id": 0, "Ev Idx": 3177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313596.351, "dur": 5.000,
    "args": {
      "External id": 2469483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313596.816, "dur": 0.982,
    "args": {
      "External id": 2469484,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313597.103, "dur": 0.604,
    "args": {
      "External id": 2469485,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313604.828, "dur": 5.310,
    "args": {
      "External id": 2469486,"Record function id": 0, "Ev Idx": 3181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313605.925, "dur": 3.805,
    "args": {
      "External id": 2469487,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313606.369, "dur": 2.968,
    "args": {
      "External id": 2469488,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313606.627, "dur": 2.643,
    "args": {
      "External id": 2469489,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313613.586, "dur": 4.044,
    "args": {
      "External id": 2469490,"Record function id": 0, "Ev Idx": 3185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313614.647, "dur": 2.539,
    "args": {
      "External id": 2469491,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313615.310, "dur": 1.407,
    "args": {
      "External id": 2469492,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313615.969, "dur": 0.652,
    "args": {
      "External id": 2469493,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313621.653, "dur": 36295.179,
    "args": {
      "External id": 2469494,"Record function id": 0, "Sequence number": 24740455, "Fwd thread id": 1, "Ev Idx": 3189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313623.126, "dur": 36285.252,
    "args": {
      "External id": 2469495,"Sequence number": 24740455, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3190
    }
  },
  {
    "ph": "f", "id": 193, "pid": 1336756, "tid": 1381189, "ts": 1514657313623.126,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.17)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313654.473, "dur": 42.478,
    "args": {
      "External id": 2469496,"Record function id": 0, "Ev Idx": 3191
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.17)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313704.882, "dur": 70.390,
    "args": {
      "External id": 2469497,"Record function id": 0, "Ev Idx": 3192
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.17)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313781.716, "dur": 36118.065,
    "args": {
      "External id": 2469498,"Record function id": 0, "Ev Idx": 3193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313867.815, "dur": 6.498,
    "args": {
      "External id": 2469499,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313887.295, "dur": 4.779,
    "args": {
      "External id": 2469500,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313906.486, "dur": 35174.019,
    "args": {
      "External id": 2469501,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313919.975, "dur": 35151.655,
    "args": {
      "External id": 2469502,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657313965.595, "dur": 13.676,
    "args": {
      "External id": 2469503,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657314023.955, "dur": 35003.635,
    "args": {
      "External id": 2469504,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657314026.951, "dur": 34999.805,
    "args": {
      "External id": 2469505,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657314031.508, "dur": 7.032,
    "args": {
      "External id": 2469506,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657314040.753, "dur": 34982.753,
    "args": {
      "External id": 2469507,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349172.164, "dur": 8.534,
    "args": {
      "External id": 2469508,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349175.346, "dur": 5.054,
    "args": {
      "External id": 2469509,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349212.026, "dur": 377.541,
    "args": {
      "External id": 2469510,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349238.575, "dur": 345.678,
    "args": {
      "External id": 2469511,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3206, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349249.557, "dur": 328.479,
    "args": {
      "External id": 2469512,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349610.645, "dur": 2.390,
    "args": {
      "External id": 2469513,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3208, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349673.257, "dur": 6.817,
    "args": {
      "External id": 2469514,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349728.359, "dur": 1.155,
    "args": {
      "External id": 2469515,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349747.011, "dur": 0.997,
    "args": {
      "External id": 2469516,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349760.576, "dur": 0.914,
    "args": {
      "External id": 2469517,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349773.062, "dur": 0.938,
    "args": {
      "External id": 2469518,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349788.455, "dur": 1.224,
    "args": {
      "External id": 2469519,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349801.656, "dur": 0.937,
    "args": {
      "External id": 2469520,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349819.109, "dur": 2.024,
    "args": {
      "External id": 2469521,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349832.990, "dur": 0.878,
    "args": {
      "External id": 2469522,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349930.652, "dur": 2768.964,
    "args": {
      "External id": 2469523,"Record function id": 0, "Ev Idx": 3218
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.16)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349949.231, "dur": 1017.142,
    "args": {
      "External id": 2469524,"Record function id": 0, "Ev Idx": 3219
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657349963.167, "dur": 355.525,
    "args": {
      "External id": 2469525,"Record function id": 0, "Ev Idx": 3220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350082.403, "dur": 5.013,
    "args": {
      "External id": 2469526,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350090.716, "dur": 1.163,
    "args": {
      "External id": 2469527,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350093.922, "dur": 0.922,
    "args": {
      "External id": 2469528,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350096.469, "dur": 0.919,
    "args": {
      "External id": 2469529,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350098.752, "dur": 0.607,
    "args": {
      "External id": 2469530,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350100.898, "dur": 1.182,
    "args": {
      "External id": 2469531,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350103.618, "dur": 1.899,
    "args": {
      "External id": 2469532,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350106.990, "dur": 2.834,
    "args": {
      "External id": 2469533,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350111.147, "dur": 0.823,
    "args": {
      "External id": 2469534,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350113.323, "dur": 0.851,
    "args": {
      "External id": 2469535,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350133.200, "dur": 153.166,
    "args": {
      "External id": 2469536,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350149.285, "dur": 132.918,
    "args": {
      "External id": 2469537,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350172.334, "dur": 11.705,
    "args": {
      "External id": 2469538,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350187.705, "dur": 64.670,
    "args": {
      "External id": 2469539,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350190.322, "dur": 61.708,
    "args": {
      "External id": 2469540,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350194.780, "dur": 4.897,
    "args": {
      "External id": 2469541,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350201.604, "dur": 49.653,
    "args": {
      "External id": 2469542,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3237
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.15", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350403.608, "dur": 554.072,
    "args": {
      "External id": 2469543,"Record function id": 0, "Ev Idx": 3238
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350420.866, "dur": 523.828,
    "args": {
      "External id": 2469544,"Record function id": 0, "Ev Idx": 3239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350476.049, "dur": 4.233,
    "args": {
      "External id": 2469545,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350495.465, "dur": 31.448,
    "args": {
      "External id": 2469546,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350500.732, "dur": 1.600,
    "args": {
      "External id": 2469547,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350504.424, "dur": 0.478,
    "args": {
      "External id": 2469548,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350506.580, "dur": 2.751,
    "args": {
      "External id": 2469549,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350511.126, "dur": 0.410,
    "args": {
      "External id": 2469550,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350513.511, "dur": 0.456,
    "args": {
      "External id": 2469551,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350515.823, "dur": 0.398,
    "args": {
      "External id": 2469552,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350517.900, "dur": 0.431,
    "args": {
      "External id": 2469553,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350520.335, "dur": 0.622,
    "args": {
      "External id": 2469554,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350522.530, "dur": 0.595,
    "args": {
      "External id": 2469555,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350555.049, "dur": 34.875,
    "args": {
      "External id": 2469556,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350622.324, "dur": 106.200,
    "args": {
      "External id": 2469557,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350632.948, "dur": 4.178,
    "args": {
      "External id": 2469558,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350642.346, "dur": 12.600,
    "args": {
      "External id": 2469559,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350646.764, "dur": 7.746,
    "args": {
      "External id": 2469560,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350650.788, "dur": 2.287,
    "args": {
      "External id": 2469561,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350662.194, "dur": 26.025,
    "args": {
      "External id": 2469562,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350664.866, "dur": 0.454,
    "args": {
      "External id": 2469563,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350667.204, "dur": 0.352,
    "args": {
      "External id": 2469564,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350669.176, "dur": 0.394,
    "args": {
      "External id": 2469565,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350671.163, "dur": 0.574,
    "args": {
      "External id": 2469566,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350673.428, "dur": 0.370,
    "args": {
      "External id": 2469567,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350675.894, "dur": 0.386,
    "args": {
      "External id": 2469568,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350677.695, "dur": 0.321,
    "args": {
      "External id": 2469569,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350679.853, "dur": 2.590,
    "args": {
      "External id": 2469570,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350684.166, "dur": 0.418,
    "args": {
      "External id": 2469571,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350700.259, "dur": 20.946,
    "args": {
      "External id": 2469572,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350772.874, "dur": 109.178,
    "args": {
      "External id": 2469573,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350794.564, "dur": 83.915,
    "args": {
      "External id": 2469574,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3269, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350804.051, "dur": 70.455,
    "args": {
      "External id": 2469575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350895.057, "dur": 1.739,
    "args": {
      "External id": 2469576,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3271, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657350973.308, "dur": 1705.652,
    "args": {
      "External id": 2469577,"Sequence number": 24740454, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3272
    }
  },
  {
    "ph": "f", "id": 194, "pid": 1336756, "tid": 1381189, "ts": 1514657350973.308,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351128.146, "dur": 106.822,
    "args": {
      "External id": 2469578,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351273.496, "dur": 38.215,
    "args": {
      "External id": 2469579,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351331.259, "dur": 49.671,
    "args": {
      "External id": 2469580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351390.841, "dur": 33.147,
    "args": {
      "External id": 2469581,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351431.827, "dur": 50.052,
    "args": {
      "External id": 2469582,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351510.451, "dur": 45.407,
    "args": {
      "External id": 2469583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351569.127, "dur": 48.192,
    "args": {
      "External id": 2469584,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351642.051, "dur": 24.739,
    "args": {
      "External id": 2469585,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351686.041, "dur": 30.219,
    "args": {
      "External id": 2469586,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351736.614, "dur": 21.029,
    "args": {
      "External id": 2469587,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351769.610, "dur": 15.672,
    "args": {
      "External id": 2469588,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351796.294, "dur": 30.698,
    "args": {
      "External id": 2469589,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351830.132, "dur": 32.844,
    "args": {
      "External id": 2469590,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351890.877, "dur": 213.253,
    "args": {
      "External id": 2469591,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351968.550, "dur": 6.329,
    "args": {
      "External id": 2469592,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657351977.047, "dur": 2.241,
    "args": {
      "External id": 2469593,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352139.514, "dur": 24.849,
    "args": {
      "External id": 2469594,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352176.098, "dur": 17.192,
    "args": {
      "External id": 2469595,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352205.358, "dur": 42.364,
    "args": {
      "External id": 2469596,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352254.594, "dur": 35.634,
    "args": {
      "External id": 2469597,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352297.437, "dur": 22.415,
    "args": {
      "External id": 2469598,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352325.727, "dur": 31.634,
    "args": {
      "External id": 2469599,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352365.553, "dur": 21.046,
    "args": {
      "External id": 2469600,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352392.783, "dur": 43.732,
    "args": {
      "External id": 2469601,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352459.077, "dur": 25.437,
    "args": {
      "External id": 2469602,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352500.849, "dur": 22.611,
    "args": {
      "External id": 2469603,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352558.611, "dur": 21.472,
    "args": {
      "External id": 2469604,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352596.814, "dur": 19.213,
    "args": {
      "External id": 2469605,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352629.013, "dur": 17.965,
    "args": {
      "External id": 2469606,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352724.930, "dur": 15.460,
    "args": {
      "External id": 2469607,"Record function id": 0, "Ev Idx": 3302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352728.299, "dur": 11.164,
    "args": {
      "External id": 2469608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352732.514, "dur": 6.134,
    "args": {
      "External id": 2469609,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352734.336, "dur": 4.199,
    "args": {
      "External id": 2469610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352744.343, "dur": 4.807,
    "args": {
      "External id": 2469611,"Record function id": 0, "Ev Idx": 3306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352745.599, "dur": 3.092,
    "args": {
      "External id": 2469612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352746.519, "dur": 1.718,
    "args": {
      "External id": 2469613,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352747.073, "dur": 1.076,
    "args": {
      "External id": 2469614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352752.332, "dur": 4.414,
    "args": {
      "External id": 2469615,"Record function id": 0, "Ev Idx": 3310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352753.676, "dur": 2.690,
    "args": {
      "External id": 2469616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352754.579, "dur": 1.360,
    "args": {
      "External id": 2469617,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352755.114, "dur": 0.752,
    "args": {
      "External id": 2469618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352759.963, "dur": 4.354,
    "args": {
      "External id": 2469619,"Record function id": 0, "Ev Idx": 3314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352761.681, "dur": 2.246,
    "args": {
      "External id": 2469620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352762.517, "dur": 1.017,
    "args": {
      "External id": 2469621,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352762.814, "dur": 0.653,
    "args": {
      "External id": 2469622,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352767.788, "dur": 5.318,
    "args": {
      "External id": 2469623,"Record function id": 0, "Ev Idx": 3318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352769.497, "dur": 3.208,
    "args": {
      "External id": 2469624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352770.507, "dur": 1.786,
    "args": {
      "External id": 2469625,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352771.474, "dur": 0.672,
    "args": {
      "External id": 2469626,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352776.493, "dur": 4.220,
    "args": {
      "External id": 2469627,"Record function id": 0, "Ev Idx": 3322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352777.519, "dur": 2.816,
    "args": {
      "External id": 2469628,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352778.474, "dur": 1.460,
    "args": {
      "External id": 2469629,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352779.034, "dur": 0.836,
    "args": {
      "External id": 2469630,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352784.132, "dur": 5.797,
    "args": {
      "External id": 2469631,"Record function id": 0, "Ev Idx": 3326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352785.211, "dur": 4.319,
    "args": {
      "External id": 2469632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352785.710, "dur": 3.419,
    "args": {
      "External id": 2469633,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352786.138, "dur": 2.915,
    "args": {
      "External id": 2469634,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352793.301, "dur": 4.358,
    "args": {
      "External id": 2469635,"Record function id": 0, "Ev Idx": 3330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352794.563, "dur": 2.687,
    "args": {
      "External id": 2469636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352795.185, "dur": 1.660,
    "args": {
      "External id": 2469637,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352795.764, "dur": 0.966,
    "args": {
      "External id": 2469638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352800.858, "dur": 3.650,
    "args": {
      "External id": 2469639,"Record function id": 0, "Ev Idx": 3334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352801.868, "dur": 2.257,
    "args": {
      "External id": 2469640,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352802.534, "dur": 1.194,
    "args": {
      "External id": 2469641,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352802.975, "dur": 0.659,
    "args": {
      "External id": 2469642,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352808.190, "dur": 37164.756,
    "args": {
      "External id": 2469643,"Record function id": 0, "Sequence number": 24740453, "Fwd thread id": 1, "Ev Idx": 3338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352809.870, "dur": 37154.149,
    "args": {
      "External id": 2469644,"Sequence number": 24740453, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3339
    }
  },
  {
    "ph": "f", "id": 195, "pid": 1336756, "tid": 1381189, "ts": 1514657352809.870,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.16)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352837.644, "dur": 39.194,
    "args": {
      "External id": 2469645,"Record function id": 0, "Ev Idx": 3340
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.16)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352883.812, "dur": 62.180,
    "args": {
      "External id": 2469646,"Record function id": 0, "Ev Idx": 3341
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.16)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657352951.455, "dur": 37004.454,
    "args": {
      "External id": 2469647,"Record function id": 0, "Ev Idx": 3342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353079.821, "dur": 7.204,
    "args": {
      "External id": 2469648,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353097.666, "dur": 5.083,
    "args": {
      "External id": 2469649,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353122.620, "dur": 35982.098,
    "args": {
      "External id": 2469650,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353135.969, "dur": 35960.276,
    "args": {
      "External id": 2469651,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353180.107, "dur": 14.185,
    "args": {
      "External id": 2469652,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353200.733, "dur": 35856.565,
    "args": {
      "External id": 2469653,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353203.259, "dur": 35853.408,
    "args": {
      "External id": 2469654,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353207.654, "dur": 4.949,
    "args": {
      "External id": 2469655,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657353214.510, "dur": 35838.414,
    "args": {
      "External id": 2469656,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389193.515, "dur": 8.622,
    "args": {
      "External id": 2469657,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389196.473, "dur": 5.373,
    "args": {
      "External id": 2469658,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389232.505, "dur": 407.822,
    "args": {
      "External id": 2469659,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389261.843, "dur": 373.239,
    "args": {
      "External id": 2469660,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3355, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389273.085, "dur": 355.807,
    "args": {
      "External id": 2469661,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389673.536, "dur": 2.236,
    "args": {
      "External id": 2469662,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3357, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389735.530, "dur": 8.730,
    "args": {
      "External id": 2469663,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389789.668, "dur": 1.514,
    "args": {
      "External id": 2469664,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389808.019, "dur": 1.189,
    "args": {
      "External id": 2469665,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389822.822, "dur": 0.956,
    "args": {
      "External id": 2469666,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389836.128, "dur": 2.427,
    "args": {
      "External id": 2469667,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389850.342, "dur": 1.017,
    "args": {
      "External id": 2469668,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389863.060, "dur": 1.015,
    "args": {
      "External id": 2469669,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389877.450, "dur": 2.174,
    "args": {
      "External id": 2469670,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657389891.666, "dur": 2.988,
    "args": {
      "External id": 2469671,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390023.034, "dur": 2768.265,
    "args": {
      "External id": 2469672,"Record function id": 0, "Ev Idx": 3367
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.15)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390046.223, "dur": 1028.014,
    "args": {
      "External id": 2469673,"Record function id": 0, "Ev Idx": 3368
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390061.458, "dur": 322.174,
    "args": {
      "External id": 2469674,"Record function id": 0, "Ev Idx": 3369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390142.818, "dur": 4.446,
    "args": {
      "External id": 2469675,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390151.106, "dur": 1.189,
    "args": {
      "External id": 2469676,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390154.392, "dur": 1.243,
    "args": {
      "External id": 2469677,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390157.231, "dur": 0.996,
    "args": {
      "External id": 2469678,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390160.018, "dur": 0.721,
    "args": {
      "External id": 2469679,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390162.537, "dur": 0.787,
    "args": {
      "External id": 2469680,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390165.104, "dur": 3.587,
    "args": {
      "External id": 2469681,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390170.316, "dur": 1.119,
    "args": {
      "External id": 2469682,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390173.041, "dur": 0.645,
    "args": {
      "External id": 2469683,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390175.295, "dur": 0.894,
    "args": {
      "External id": 2469684,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390194.337, "dur": 156.334,
    "args": {
      "External id": 2469685,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390211.140, "dur": 134.967,
    "args": {
      "External id": 2469686,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390231.876, "dur": 12.385,
    "args": {
      "External id": 2469687,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390247.642, "dur": 66.515,
    "args": {
      "External id": 2469688,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390250.555, "dur": 63.278,
    "args": {
      "External id": 2469689,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390255.248, "dur": 5.022,
    "args": {
      "External id": 2469690,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390262.339, "dur": 50.712,
    "args": {
      "External id": 2469691,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3386
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.14", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390467.667, "dur": 598.322,
    "args": {
      "External id": 2469692,"Record function id": 0, "Ev Idx": 3387
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390485.322, "dur": 566.921,
    "args": {
      "External id": 2469693,"Record function id": 0, "Ev Idx": 3388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390557.852, "dur": 6.492,
    "args": {
      "External id": 2469694,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390580.123, "dur": 32.864,
    "args": {
      "External id": 2469695,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390585.767, "dur": 1.498,
    "args": {
      "External id": 2469696,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390589.927, "dur": 2.273,
    "args": {
      "External id": 2469697,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390594.172, "dur": 0.634,
    "args": {
      "External id": 2469698,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390596.491, "dur": 0.488,
    "args": {
      "External id": 2469699,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390598.855, "dur": 0.404,
    "args": {
      "External id": 2469700,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390601.137, "dur": 0.383,
    "args": {
      "External id": 2469701,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390603.281, "dur": 0.507,
    "args": {
      "External id": 2469702,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390605.637, "dur": 0.356,
    "args": {
      "External id": 2469703,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390608.402, "dur": 0.328,
    "args": {
      "External id": 2469704,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390623.790, "dur": 33.145,
    "args": {
      "External id": 2469705,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390688.532, "dur": 102.476,
    "args": {
      "External id": 2469706,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390699.264, "dur": 4.451,
    "args": {
      "External id": 2469707,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390708.956, "dur": 10.491,
    "args": {
      "External id": 2469708,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390713.126, "dur": 5.887,
    "args": {
      "External id": 2469709,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390717.383, "dur": 0.564,
    "args": {
      "External id": 2469710,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390726.418, "dur": 26.560,
    "args": {
      "External id": 2469711,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390728.909, "dur": 0.474,
    "args": {
      "External id": 2469712,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390731.651, "dur": 0.396,
    "args": {
      "External id": 2469713,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390734.194, "dur": 0.506,
    "args": {
      "External id": 2469714,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390736.514, "dur": 0.390,
    "args": {
      "External id": 2469715,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390738.954, "dur": 0.446,
    "args": {
      "External id": 2469716,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390741.018, "dur": 0.352,
    "args": {
      "External id": 2469717,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390743.424, "dur": 2.052,
    "args": {
      "External id": 2469718,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390746.963, "dur": 0.353,
    "args": {
      "External id": 2469719,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390749.227, "dur": 0.433,
    "args": {
      "External id": 2469720,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390762.410, "dur": 21.021,
    "args": {
      "External id": 2469721,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390837.046, "dur": 109.995,
    "args": {
      "External id": 2469722,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390859.159, "dur": 84.248,
    "args": {
      "External id": 2469723,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3418, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390869.272, "dur": 70.174,
    "args": {
      "External id": 2469724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657390960.779, "dur": 1.787,
    "args": {
      "External id": 2469725,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3420, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391082.188, "dur": 1688.253,
    "args": {
      "External id": 2469726,"Sequence number": 24740452, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3421
    }
  },
  {
    "ph": "f", "id": 196, "pid": 1336756, "tid": 1381189, "ts": 1514657391082.188,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391190.637, "dur": 107.162,
    "args": {
      "External id": 2469727,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391335.981, "dur": 38.551,
    "args": {
      "External id": 2469728,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391393.316, "dur": 51.702,
    "args": {
      "External id": 2469729,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391454.821, "dur": 34.027,
    "args": {
      "External id": 2469730,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391495.765, "dur": 63.168,
    "args": {
      "External id": 2469731,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391571.444, "dur": 32.060,
    "args": {
      "External id": 2469732,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391610.650, "dur": 43.660,
    "args": {
      "External id": 2469733,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391677.876, "dur": 24.520,
    "args": {
      "External id": 2469734,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391721.139, "dur": 32.684,
    "args": {
      "External id": 2469735,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391772.000, "dur": 19.966,
    "args": {
      "External id": 2469736,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391806.858, "dur": 15.845,
    "args": {
      "External id": 2469737,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391832.389, "dur": 29.031,
    "args": {
      "External id": 2469738,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391864.910, "dur": 33.163,
    "args": {
      "External id": 2469739,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657391938.954, "dur": 229.280,
    "args": {
      "External id": 2469740,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392064.472, "dur": 6.894,
    "args": {
      "External id": 2469741,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392074.285, "dur": 3.829,
    "args": {
      "External id": 2469742,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392202.661, "dur": 26.229,
    "args": {
      "External id": 2469743,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392241.285, "dur": 17.060,
    "args": {
      "External id": 2469744,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392268.023, "dur": 45.552,
    "args": {
      "External id": 2469745,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392321.465, "dur": 39.653,
    "args": {
      "External id": 2469746,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392368.557, "dur": 22.256,
    "args": {
      "External id": 2469747,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392398.829, "dur": 32.923,
    "args": {
      "External id": 2469748,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392451.297, "dur": 30.919,
    "args": {
      "External id": 2469749,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392490.668, "dur": 53.241,
    "args": {
      "External id": 2469750,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392571.596, "dur": 27.399,
    "args": {
      "External id": 2469751,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392620.806, "dur": 24.403,
    "args": {
      "External id": 2469752,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392659.573, "dur": 19.395,
    "args": {
      "External id": 2469753,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392694.832, "dur": 15.247,
    "args": {
      "External id": 2469754,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392724.853, "dur": 17.424,
    "args": {
      "External id": 2469755,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392814.476, "dur": 15.963,
    "args": {
      "External id": 2469756,"Record function id": 0, "Ev Idx": 3451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392817.984, "dur": 11.584,
    "args": {
      "External id": 2469757,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392822.267, "dur": 6.404,
    "args": {
      "External id": 2469758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392824.302, "dur": 4.209,
    "args": {
      "External id": 2469759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392838.378, "dur": 5.588,
    "args": {
      "External id": 2469760,"Record function id": 0, "Ev Idx": 3455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392840.149, "dur": 3.392,
    "args": {
      "External id": 2469761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392840.939, "dur": 2.184,
    "args": {
      "External id": 2469762,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392842.042, "dur": 0.981,
    "args": {
      "External id": 2469763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392847.145, "dur": 4.769,
    "args": {
      "External id": 2469764,"Record function id": 0, "Ev Idx": 3459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392848.585, "dur": 2.922,
    "args": {
      "External id": 2469765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392849.435, "dur": 1.645,
    "args": {
      "External id": 2469766,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392849.952, "dur": 1.051,
    "args": {
      "External id": 2469767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392855.037, "dur": 5.843,
    "args": {
      "External id": 2469768,"Record function id": 0, "Ev Idx": 3463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392856.200, "dur": 4.276,
    "args": {
      "External id": 2469769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392857.193, "dur": 2.876,
    "args": {
      "External id": 2469770,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392857.678, "dur": 2.320,
    "args": {
      "External id": 2469771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392863.931, "dur": 4.454,
    "args": {
      "External id": 2469772,"Record function id": 0, "Ev Idx": 3467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392865.384, "dur": 2.595,
    "args": {
      "External id": 2469773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392866.170, "dur": 1.396,
    "args": {
      "External id": 2469774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392866.445, "dur": 1.030,
    "args": {
      "External id": 2469775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392871.655, "dur": 3.802,
    "args": {
      "External id": 2469776,"Record function id": 0, "Ev Idx": 3471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392872.756, "dur": 2.270,
    "args": {
      "External id": 2469777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392873.506, "dur": 1.134,
    "args": {
      "External id": 2469778,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392873.928, "dur": 0.605,
    "args": {
      "External id": 2469779,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392878.679, "dur": 3.767,
    "args": {
      "External id": 2469780,"Record function id": 0, "Ev Idx": 3475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392879.936, "dur": 2.061,
    "args": {
      "External id": 2469781,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392880.475, "dur": 1.138,
    "args": {
      "External id": 2469782,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392880.942, "dur": 0.573,
    "args": {
      "External id": 2469783,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392885.504, "dur": 3.955,
    "args": {
      "External id": 2469784,"Record function id": 0, "Ev Idx": 3479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392886.878, "dur": 2.135,
    "args": {
      "External id": 2469785,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392887.359, "dur": 1.270,
    "args": {
      "External id": 2469786,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392887.855, "dur": 0.658,
    "args": {
      "External id": 2469787,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392896.285, "dur": 3.675,
    "args": {
      "External id": 2469788,"Record function id": 0, "Ev Idx": 3483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392897.398, "dur": 2.120,
    "args": {
      "External id": 2469789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392897.878, "dur": 1.248,
    "args": {
      "External id": 2469790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392898.397, "dur": 0.611,
    "args": {
      "External id": 2469791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392903.912, "dur": 36224.010,
    "args": {
      "External id": 2469792,"Record function id": 0, "Sequence number": 24740451, "Fwd thread id": 1, "Ev Idx": 3487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392905.320, "dur": 36214.075,
    "args": {
      "External id": 2469793,"Sequence number": 24740451, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3488
    }
  },
  {
    "ph": "f", "id": 197, "pid": 1336756, "tid": 1381189, "ts": 1514657392905.320,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.15)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657392933.450, "dur": 40.558,
    "args": {
      "External id": 2469794,"Record function id": 0, "Ev Idx": 3489
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.15)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393021.193, "dur": 70.930,
    "args": {
      "External id": 2469795,"Record function id": 0, "Ev Idx": 3490
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.15)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393099.255, "dur": 36011.796,
    "args": {
      "External id": 2469796,"Record function id": 0, "Ev Idx": 3491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393190.811, "dur": 6.949,
    "args": {
      "External id": 2469797,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393212.049, "dur": 5.039,
    "args": {
      "External id": 2469798,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393231.620, "dur": 35034.487,
    "args": {
      "External id": 2469799,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393245.298, "dur": 35012.422,
    "args": {
      "External id": 2469800,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393291.792, "dur": 15.511,
    "args": {
      "External id": 2469801,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393314.043, "dur": 34903.874,
    "args": {
      "External id": 2469802,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393317.012, "dur": 34900.282,
    "args": {
      "External id": 2469803,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393320.958, "dur": 5.560,
    "args": {
      "External id": 2469804,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657393328.515, "dur": 34885.198,
    "args": {
      "External id": 2469805,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428355.278, "dur": 9.268,
    "args": {
      "External id": 2469806,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428358.929, "dur": 5.244,
    "args": {
      "External id": 2469807,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428395.179, "dur": 382.322,
    "args": {
      "External id": 2469808,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428429.439, "dur": 342.836,
    "args": {
      "External id": 2469809,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3504, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428440.673, "dur": 325.706,
    "args": {
      "External id": 2469810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428798.732, "dur": 2.098,
    "args": {
      "External id": 2469811,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3506, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428861.079, "dur": 6.491,
    "args": {
      "External id": 2469812,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428913.626, "dur": 1.676,
    "args": {
      "External id": 2469813,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428931.495, "dur": 0.941,
    "args": {
      "External id": 2469814,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428946.373, "dur": 0.864,
    "args": {
      "External id": 2469815,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428958.547, "dur": 1.054,
    "args": {
      "External id": 2469816,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657428970.555, "dur": 0.923,
    "args": {
      "External id": 2469817,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429015.946, "dur": 1.753,
    "args": {
      "External id": 2469818,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429035.695, "dur": 2.190,
    "args": {
      "External id": 2469819,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429048.351, "dur": 0.950,
    "args": {
      "External id": 2469820,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429144.079, "dur": 2792.789,
    "args": {
      "External id": 2469821,"Record function id": 0, "Ev Idx": 3516
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.14)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429163.277, "dur": 1040.289,
    "args": {
      "External id": 2469822,"Record function id": 0, "Ev Idx": 3517
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429178.028, "dur": 323.940,
    "args": {
      "External id": 2469823,"Record function id": 0, "Ev Idx": 3518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429259.949, "dur": 4.256,
    "args": {
      "External id": 2469824,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429267.331, "dur": 1.111,
    "args": {
      "External id": 2469825,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429270.514, "dur": 1.502,
    "args": {
      "External id": 2469826,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429273.847, "dur": 2.298,
    "args": {
      "External id": 2469827,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429277.703, "dur": 0.694,
    "args": {
      "External id": 2469828,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429280.028, "dur": 1.219,
    "args": {
      "External id": 2469829,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429282.661, "dur": 2.041,
    "args": {
      "External id": 2469830,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429286.087, "dur": 3.979,
    "args": {
      "External id": 2469831,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429291.431, "dur": 0.717,
    "args": {
      "External id": 2469832,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429293.458, "dur": 0.903,
    "args": {
      "External id": 2469833,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429313.079, "dur": 156.945,
    "args": {
      "External id": 2469834,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429329.167, "dur": 136.405,
    "args": {
      "External id": 2469835,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429351.477, "dur": 12.187,
    "args": {
      "External id": 2469836,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429367.076, "dur": 68.330,
    "args": {
      "External id": 2469837,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429369.676, "dur": 65.332,
    "args": {
      "External id": 2469838,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429373.829, "dur": 7.686,
    "args": {
      "External id": 2469839,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429383.521, "dur": 50.698,
    "args": {
      "External id": 2469840,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3535
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.13", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429602.691, "dur": 592.592,
    "args": {
      "External id": 2469841,"Record function id": 0, "Ev Idx": 3536
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429620.047, "dur": 562.143,
    "args": {
      "External id": 2469842,"Record function id": 0, "Ev Idx": 3537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429678.002, "dur": 5.618,
    "args": {
      "External id": 2469843,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429698.526, "dur": 30.432,
    "args": {
      "External id": 2469844,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429703.548, "dur": 1.358,
    "args": {
      "External id": 2469845,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429706.985, "dur": 0.640,
    "args": {
      "External id": 2469846,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429709.226, "dur": 0.672,
    "args": {
      "External id": 2469847,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429711.405, "dur": 0.606,
    "args": {
      "External id": 2469848,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429713.534, "dur": 0.375,
    "args": {
      "External id": 2469849,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429715.841, "dur": 0.659,
    "args": {
      "External id": 2469850,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429718.296, "dur": 2.469,
    "args": {
      "External id": 2469851,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429722.500, "dur": 0.420,
    "args": {
      "External id": 2469852,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429724.451, "dur": 0.381,
    "args": {
      "External id": 2469853,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429739.709, "dur": 33.827,
    "args": {
      "External id": 2469854,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429805.081, "dur": 106.969,
    "args": {
      "External id": 2469855,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429815.391, "dur": 3.170,
    "args": {
      "External id": 2469856,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429828.904, "dur": 10.445,
    "args": {
      "External id": 2469857,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429833.230, "dur": 5.689,
    "args": {
      "External id": 2469858,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429837.250, "dur": 0.522,
    "args": {
      "External id": 2469859,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429847.979, "dur": 24.696,
    "args": {
      "External id": 2469860,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429850.227, "dur": 0.414,
    "args": {
      "External id": 2469861,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429852.362, "dur": 0.437,
    "args": {
      "External id": 2469862,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429854.306, "dur": 0.386,
    "args": {
      "External id": 2469863,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429856.419, "dur": 2.296,
    "args": {
      "External id": 2469864,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429860.308, "dur": 0.469,
    "args": {
      "External id": 2469865,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429862.808, "dur": 0.396,
    "args": {
      "External id": 2469866,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429865.002, "dur": 0.318,
    "args": {
      "External id": 2469867,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429866.981, "dur": 0.442,
    "args": {
      "External id": 2469868,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429869.031, "dur": 0.424,
    "args": {
      "External id": 2469869,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429884.406, "dur": 20.473,
    "args": {
      "External id": 2469870,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429957.797, "dur": 152.257,
    "args": {
      "External id": 2469871,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657429979.633, "dur": 126.280,
    "args": {
      "External id": 2469872,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3567, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430025.723, "dur": 75.630,
    "args": {
      "External id": 2469873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430125.384, "dur": 1.817,
    "args": {
      "External id": 2469874,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3569, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430211.316, "dur": 1705.123,
    "args": {
      "External id": 2469875,"Sequence number": 24740450, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3570
    }
  },
  {
    "ph": "f", "id": 198, "pid": 1336756, "tid": 1381189, "ts": 1514657430211.316,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430323.286, "dur": 104.485,
    "args": {
      "External id": 2469876,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430465.040, "dur": 38.066,
    "args": {
      "External id": 2469877,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430517.955, "dur": 70.154,
    "args": {
      "External id": 2469878,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430600.822, "dur": 34.989,
    "args": {
      "External id": 2469879,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430645.102, "dur": 48.228,
    "args": {
      "External id": 2469880,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430700.456, "dur": 28.073,
    "args": {
      "External id": 2469881,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430736.198, "dur": 43.003,
    "args": {
      "External id": 2469882,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430800.829, "dur": 23.584,
    "args": {
      "External id": 2469883,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430842.339, "dur": 32.670,
    "args": {
      "External id": 2469884,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430892.284, "dur": 23.333,
    "args": {
      "External id": 2469885,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430929.107, "dur": 14.744,
    "args": {
      "External id": 2469886,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657430952.486, "dur": 76.594,
    "args": {
      "External id": 2469887,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431034.938, "dur": 41.787,
    "args": {
      "External id": 2469888,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431133.760, "dur": 179.827,
    "args": {
      "External id": 2469889,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431217.539, "dur": 5.576,
    "args": {
      "External id": 2469890,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431225.236, "dur": 2.080,
    "args": {
      "External id": 2469891,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431347.085, "dur": 25.942,
    "args": {
      "External id": 2469892,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431383.513, "dur": 17.256,
    "args": {
      "External id": 2469893,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431413.098, "dur": 38.084,
    "args": {
      "External id": 2469894,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431458.111, "dur": 37.228,
    "args": {
      "External id": 2469895,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431503.831, "dur": 22.179,
    "args": {
      "External id": 2469896,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431549.864, "dur": 37.704,
    "args": {
      "External id": 2469897,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431611.139, "dur": 31.513,
    "args": {
      "External id": 2469898,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431650.710, "dur": 35.968,
    "args": {
      "External id": 2469899,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431706.604, "dur": 27.398,
    "args": {
      "External id": 2469900,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431760.529, "dur": 26.838,
    "args": {
      "External id": 2469901,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431803.831, "dur": 19.253,
    "args": {
      "External id": 2469902,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431841.946, "dur": 16.129,
    "args": {
      "External id": 2469903,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431870.204, "dur": 17.699,
    "args": {
      "External id": 2469904,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431959.870, "dur": 15.082,
    "args": {
      "External id": 2469905,"Record function id": 0, "Ev Idx": 3600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431962.942, "dur": 11.055,
    "args": {
      "External id": 2469906,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431967.016, "dur": 6.154,
    "args": {
      "External id": 2469907,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431968.719, "dur": 4.315,
    "args": {
      "External id": 2469908,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431978.664, "dur": 42.337,
    "args": {
      "External id": 2469909,"Record function id": 0, "Ev Idx": 3604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657431980.117, "dur": 39.849,
    "args": {
      "External id": 2469910,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432015.789, "dur": 3.198,
    "args": {
      "External id": 2469911,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432016.813, "dur": 1.857,
    "args": {
      "External id": 2469912,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432026.133, "dur": 7.119,
    "args": {
      "External id": 2469913,"Record function id": 0, "Ev Idx": 3608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432027.799, "dur": 4.988,
    "args": {
      "External id": 2469914,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432028.474, "dur": 3.812,
    "args": {
      "External id": 2469915,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432028.779, "dur": 3.422,
    "args": {
      "External id": 2469916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432036.427, "dur": 4.790,
    "args": {
      "External id": 2469917,"Record function id": 0, "Ev Idx": 3612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432038.126, "dur": 2.679,
    "args": {
      "External id": 2469918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432038.866, "dur": 1.484,
    "args": {
      "External id": 2469919,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432039.298, "dur": 0.971,
    "args": {
      "External id": 2469920,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432044.207, "dur": 4.504,
    "args": {
      "External id": 2469921,"Record function id": 0, "Ev Idx": 3616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432045.932, "dur": 2.393,
    "args": {
      "External id": 2469922,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432046.788, "dur": 1.112,
    "args": {
      "External id": 2469923,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432047.106, "dur": 0.722,
    "args": {
      "External id": 2469924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432051.843, "dur": 4.544,
    "args": {
      "External id": 2469925,"Record function id": 0, "Ev Idx": 3620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432053.007, "dur": 2.958,
    "args": {
      "External id": 2469926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432053.925, "dur": 1.527,
    "args": {
      "External id": 2469927,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432054.651, "dur": 0.607,
    "args": {
      "External id": 2469928,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432059.781, "dur": 3.856,
    "args": {
      "External id": 2469929,"Record function id": 0, "Ev Idx": 3624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432061.139, "dur": 2.097,
    "args": {
      "External id": 2469930,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432061.752, "dur": 1.045,
    "args": {
      "External id": 2469931,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432062.033, "dur": 0.672,
    "args": {
      "External id": 2469932,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432066.689, "dur": 3.574,
    "args": {
      "External id": 2469933,"Record function id": 0, "Ev Idx": 3628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432067.912, "dur": 1.932,
    "args": {
      "External id": 2469934,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432068.365, "dur": 1.060,
    "args": {
      "External id": 2469935,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432068.621, "dur": 0.732,
    "args": {
      "External id": 2469936,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432073.243, "dur": 4.374,
    "args": {
      "External id": 2469937,"Record function id": 0, "Ev Idx": 3632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432074.438, "dur": 2.792,
    "args": {
      "External id": 2469938,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432075.044, "dur": 1.759,
    "args": {
      "External id": 2469939,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432075.787, "dur": 0.921,
    "args": {
      "External id": 2469940,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432081.121, "dur": 36041.214,
    "args": {
      "External id": 2469941,"Record function id": 0, "Sequence number": 24740449, "Fwd thread id": 1, "Ev Idx": 3636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432082.583, "dur": 36030.772,
    "args": {
      "External id": 2469942,"Sequence number": 24740449, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3637
    }
  },
  {
    "ph": "f", "id": 199, "pid": 1336756, "tid": 1381189, "ts": 1514657432082.583,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.14)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432114.539, "dur": 41.057,
    "args": {
      "External id": 2469943,"Record function id": 0, "Ev Idx": 3638
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.14)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432163.130, "dur": 65.580,
    "args": {
      "External id": 2469944,"Record function id": 0, "Ev Idx": 3639
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.14)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432234.586, "dur": 35869.991,
    "args": {
      "External id": 2469945,"Record function id": 0, "Ev Idx": 3640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432325.907, "dur": 6.455,
    "args": {
      "External id": 2469946,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432342.288, "dur": 6.661,
    "args": {
      "External id": 2469947,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432366.809, "dur": 34911.961,
    "args": {
      "External id": 2469948,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432379.641, "dur": 34890.162,
    "args": {
      "External id": 2469949,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432422.960, "dur": 13.239,
    "args": {
      "External id": 2469950,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432442.883, "dur": 34785.029,
    "args": {
      "External id": 2469951,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432445.348, "dur": 34781.914,
    "args": {
      "External id": 2469952,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432449.465, "dur": 5.142,
    "args": {
      "External id": 2469953,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657432456.600, "dur": 34767.143,
    "args": {
      "External id": 2469954,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467369.077, "dur": 9.437,
    "args": {
      "External id": 2469955,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467372.524, "dur": 5.623,
    "args": {
      "External id": 2469956,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467409.365, "dur": 364.920,
    "args": {
      "External id": 2469957,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467435.278, "dur": 333.808,
    "args": {
      "External id": 2469958,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3653, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467446.139, "dur": 317.433,
    "args": {
      "External id": 2469959,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467795.378, "dur": 2.286,
    "args": {
      "External id": 2469960,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3655, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467854.641, "dur": 6.571,
    "args": {
      "External id": 2469961,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467907.888, "dur": 1.597,
    "args": {
      "External id": 2469962,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467925.291, "dur": 2.887,
    "args": {
      "External id": 2469963,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467939.657, "dur": 1.533,
    "args": {
      "External id": 2469964,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467951.506, "dur": 1.247,
    "args": {
      "External id": 2469965,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467963.724, "dur": 1.191,
    "args": {
      "External id": 2469966,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657467976.229, "dur": 2.440,
    "args": {
      "External id": 2469967,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468027.880, "dur": 2.557,
    "args": {
      "External id": 2469968,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468041.065, "dur": 1.047,
    "args": {
      "External id": 2469969,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468138.222, "dur": 2769.330,
    "args": {
      "External id": 2469970,"Record function id": 0, "Ev Idx": 3665
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.13)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468157.167, "dur": 1012.681,
    "args": {
      "External id": 2469971,"Record function id": 0, "Ev Idx": 3666
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468176.491, "dur": 320.113,
    "args": {
      "External id": 2469972,"Record function id": 0, "Ev Idx": 3667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468259.264, "dur": 3.851,
    "args": {
      "External id": 2469973,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468266.075, "dur": 1.095,
    "args": {
      "External id": 2469974,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468269.111, "dur": 3.117,
    "args": {
      "External id": 2469975,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468273.730, "dur": 1.224,
    "args": {
      "External id": 2469976,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468276.505, "dur": 0.924,
    "args": {
      "External id": 2469977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468278.920, "dur": 1.022,
    "args": {
      "External id": 2469978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468281.309, "dur": 2.022,
    "args": {
      "External id": 2469979,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468284.922, "dur": 0.798,
    "args": {
      "External id": 2469980,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468287.139, "dur": 0.894,
    "args": {
      "External id": 2469981,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468289.710, "dur": 0.790,
    "args": {
      "External id": 2469982,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468309.967, "dur": 154.791,
    "args": {
      "External id": 2469983,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468326.185, "dur": 134.213,
    "args": {
      "External id": 2469984,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468348.982, "dur": 13.513,
    "args": {
      "External id": 2469985,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468365.699, "dur": 66.017,
    "args": {
      "External id": 2469986,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468368.379, "dur": 62.996,
    "args": {
      "External id": 2469987,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468372.469, "dur": 5.730,
    "args": {
      "External id": 2469988,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468380.139, "dur": 50.512,
    "args": {
      "External id": 2469989,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3684
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.12", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468596.277, "dur": 565.661,
    "args": {
      "External id": 2469990,"Record function id": 0, "Ev Idx": 3685
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468613.905, "dur": 535.160,
    "args": {
      "External id": 2469991,"Record function id": 0, "Ev Idx": 3686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468669.981, "dur": 6.043,
    "args": {
      "External id": 2469992,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468691.193, "dur": 28.364,
    "args": {
      "External id": 2469993,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468696.171, "dur": 1.676,
    "args": {
      "External id": 2469994,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468700.000, "dur": 0.364,
    "args": {
      "External id": 2469995,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468701.925, "dur": 0.621,
    "args": {
      "External id": 2469996,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468703.835, "dur": 0.442,
    "args": {
      "External id": 2469997,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468705.915, "dur": 0.352,
    "args": {
      "External id": 2469998,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468708.049, "dur": 2.113,
    "args": {
      "External id": 2469999,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468711.466, "dur": 0.429,
    "args": {
      "External id": 2470000,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468713.355, "dur": 0.428,
    "args": {
      "External id": 2470001,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468715.475, "dur": 0.289,
    "args": {
      "External id": 2470002,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468730.595, "dur": 32.975,
    "args": {
      "External id": 2470003,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468794.055, "dur": 94.315,
    "args": {
      "External id": 2470004,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468803.929, "dur": 3.217,
    "args": {
      "External id": 2470005,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468812.129, "dur": 10.083,
    "args": {
      "External id": 2470006,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468816.386, "dur": 5.286,
    "args": {
      "External id": 2470007,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468819.907, "dur": 0.520,
    "args": {
      "External id": 2470008,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468828.299, "dur": 22.903,
    "args": {
      "External id": 2470009,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468830.347, "dur": 0.666,
    "args": {
      "External id": 2470010,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468832.717, "dur": 0.543,
    "args": {
      "External id": 2470011,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468834.856, "dur": 2.134,
    "args": {
      "External id": 2470012,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468838.526, "dur": 0.474,
    "args": {
      "External id": 2470013,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468840.506, "dur": 0.327,
    "args": {
      "External id": 2470014,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468842.297, "dur": 0.379,
    "args": {
      "External id": 2470015,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468844.426, "dur": 0.366,
    "args": {
      "External id": 2470016,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468846.372, "dur": 0.439,
    "args": {
      "External id": 2470017,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468848.062, "dur": 0.374,
    "args": {
      "External id": 2470018,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468860.426, "dur": 20.955,
    "args": {
      "External id": 2470019,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468928.716, "dur": 150.126,
    "args": {
      "External id": 2470020,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468949.495, "dur": 125.289,
    "args": {
      "External id": 2470021,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3716, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657468959.022, "dur": 111.015,
    "args": {
      "External id": 2470022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469093.907, "dur": 2.189,
    "args": {
      "External id": 2470023,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3718, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469177.968, "dur": 1709.145,
    "args": {
      "External id": 2470024,"Sequence number": 24740448, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3719
    }
  },
  {
    "ph": "f", "id": 200, "pid": 1336756, "tid": 1381189, "ts": 1514657469177.968,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469289.773, "dur": 104.626,
    "args": {
      "External id": 2470025,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469432.358, "dur": 38.375,
    "args": {
      "External id": 2470026,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469487.532, "dur": 71.489,
    "args": {
      "External id": 2470027,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469574.219, "dur": 39.420,
    "args": {
      "External id": 2470028,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469620.682, "dur": 48.324,
    "args": {
      "External id": 2470029,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469677.206, "dur": 30.703,
    "args": {
      "External id": 2470030,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469714.987, "dur": 44.350,
    "args": {
      "External id": 2470031,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469785.548, "dur": 30.612,
    "args": {
      "External id": 2470032,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469834.729, "dur": 33.744,
    "args": {
      "External id": 2470033,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469886.269, "dur": 19.372,
    "args": {
      "External id": 2470034,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469918.573, "dur": 15.931,
    "args": {
      "External id": 2470035,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469943.521, "dur": 30.321,
    "args": {
      "External id": 2470036,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657469977.258, "dur": 76.331,
    "args": {
      "External id": 2470037,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470086.399, "dur": 172.326,
    "args": {
      "External id": 2470038,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470164.970, "dur": 6.368,
    "args": {
      "External id": 2470039,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470173.292, "dur": 2.227,
    "args": {
      "External id": 2470040,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470291.527, "dur": 25.404,
    "args": {
      "External id": 2470041,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470348.070, "dur": 17.558,
    "args": {
      "External id": 2470042,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470374.582, "dur": 36.980,
    "args": {
      "External id": 2470043,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470418.411, "dur": 36.469,
    "args": {
      "External id": 2470044,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470467.768, "dur": 21.442,
    "args": {
      "External id": 2470045,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470495.295, "dur": 64.197,
    "args": {
      "External id": 2470046,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470579.421, "dur": 35.901,
    "args": {
      "External id": 2470047,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470624.141, "dur": 34.330,
    "args": {
      "External id": 2470048,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470680.242, "dur": 28.859,
    "args": {
      "External id": 2470049,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470726.324, "dur": 27.182,
    "args": {
      "External id": 2470050,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470767.073, "dur": 19.371,
    "args": {
      "External id": 2470051,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470803.010, "dur": 20.167,
    "args": {
      "External id": 2470052,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470837.772, "dur": 17.178,
    "args": {
      "External id": 2470053,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470933.938, "dur": 15.349,
    "args": {
      "External id": 2470054,"Record function id": 0, "Ev Idx": 3749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470937.538, "dur": 10.800,
    "args": {
      "External id": 2470055,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470941.588, "dur": 5.910,
    "args": {
      "External id": 2470056,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470943.457, "dur": 3.911,
    "args": {
      "External id": 2470057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470953.082, "dur": 6.791,
    "args": {
      "External id": 2470058,"Record function id": 0, "Ev Idx": 3753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470954.609, "dur": 4.823,
    "args": {
      "External id": 2470059,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470955.425, "dur": 3.470,
    "args": {
      "External id": 2470060,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470956.156, "dur": 2.613,
    "args": {
      "External id": 2470061,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470963.081, "dur": 4.068,
    "args": {
      "External id": 2470062,"Record function id": 0, "Ev Idx": 3757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470964.301, "dur": 2.455,
    "args": {
      "External id": 2470063,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470964.814, "dur": 1.504,
    "args": {
      "External id": 2470064,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470965.325, "dur": 0.911,
    "args": {
      "External id": 2470065,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470970.304, "dur": 3.505,
    "args": {
      "External id": 2470066,"Record function id": 0, "Ev Idx": 3761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470971.340, "dur": 2.048,
    "args": {
      "External id": 2470067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470971.863, "dur": 1.116,
    "args": {
      "External id": 2470068,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470972.329, "dur": 0.586,
    "args": {
      "External id": 2470069,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470977.009, "dur": 42.490,
    "args": {
      "External id": 2470070,"Record function id": 0, "Ev Idx": 3765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470977.958, "dur": 40.426,
    "args": {
      "External id": 2470071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470978.398, "dur": 38.627,
    "args": {
      "External id": 2470072,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657470978.862, "dur": 37.574,
    "args": {
      "External id": 2470073,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471025.049, "dur": 5.650,
    "args": {
      "External id": 2470074,"Record function id": 0, "Ev Idx": 3769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471026.572, "dur": 3.737,
    "args": {
      "External id": 2470075,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471027.762, "dur": 2.077,
    "args": {
      "External id": 2470076,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471028.516, "dur": 1.220,
    "args": {
      "External id": 2470077,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471033.935, "dur": 7.745,
    "args": {
      "External id": 2470078,"Record function id": 0, "Ev Idx": 3773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471035.762, "dur": 5.520,
    "args": {
      "External id": 2470079,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471036.606, "dur": 1.050,
    "args": {
      "External id": 2470080,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471036.896, "dur": 0.655,
    "args": {
      "External id": 2470081,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471044.685, "dur": 3.373,
    "args": {
      "External id": 2470082,"Record function id": 0, "Ev Idx": 3777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471045.806, "dur": 1.836,
    "args": {
      "External id": 2470083,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471046.247, "dur": 0.939,
    "args": {
      "External id": 2470084,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471046.517, "dur": 0.602,
    "args": {
      "External id": 2470085,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471051.028, "dur": 5.611,
    "args": {
      "External id": 2470086,"Record function id": 0, "Ev Idx": 3781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471052.214, "dur": 4.013,
    "args": {
      "External id": 2470087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471052.794, "dur": 2.977,
    "args": {
      "External id": 2470088,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471053.436, "dur": 2.239,
    "args": {
      "External id": 2470089,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471060.862, "dur": 38544.549,
    "args": {
      "External id": 2470090,"Record function id": 0, "Sequence number": 24740447, "Fwd thread id": 1, "Ev Idx": 3785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471062.113, "dur": 38533.949,
    "args": {
      "External id": 2470091,"Sequence number": 24740447, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3786
    }
  },
  {
    "ph": "f", "id": 201, "pid": 1336756, "tid": 1381189, "ts": 1514657471062.113,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.13)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471093.553, "dur": 42.028,
    "args": {
      "External id": 2470092,"Record function id": 0, "Ev Idx": 3787
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.13)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471143.544, "dur": 63.770,
    "args": {
      "External id": 2470093,"Record function id": 0, "Ev Idx": 3788
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.13)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471213.030, "dur": 38374.762,
    "args": {
      "External id": 2470094,"Record function id": 0, "Ev Idx": 3789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471299.133, "dur": 6.412,
    "args": {
      "External id": 2470095,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471314.983, "dur": 4.948,
    "args": {
      "External id": 2470096,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471334.683, "dur": 37351.738,
    "args": {
      "External id": 2470097,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471347.785, "dur": 37330.176,
    "args": {
      "External id": 2470098,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471387.462, "dur": 13.549,
    "args": {
      "External id": 2470099,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471407.570, "dur": 37230.583,
    "args": {
      "External id": 2470100,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471410.237, "dur": 37227.313,
    "args": {
      "External id": 2470101,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471414.052, "dur": 4.899,
    "args": {
      "External id": 2470102,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657471420.893, "dur": 37213.027,
    "args": {
      "External id": 2470103,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657508773.597, "dur": 8.610,
    "args": {
      "External id": 2470104,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657508776.866, "dur": 5.018,
    "args": {
      "External id": 2470105,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657508812.340, "dur": 447.273,
    "args": {
      "External id": 2470106,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657508839.206, "dur": 414.893,
    "args": {
      "External id": 2470107,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3802, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657508849.964, "dur": 397.237,
    "args": {
      "External id": 2470108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509282.610, "dur": 2.592,
    "args": {
      "External id": 2470109,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3804, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509348.730, "dur": 7.027,
    "args": {
      "External id": 2470110,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509401.980, "dur": 4.092,
    "args": {
      "External id": 2470111,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509423.499, "dur": 1.268,
    "args": {
      "External id": 2470112,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509438.476, "dur": 1.027,
    "args": {
      "External id": 2470113,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509454.614, "dur": 1.044,
    "args": {
      "External id": 2470114,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509467.230, "dur": 2.845,
    "args": {
      "External id": 2470115,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509480.630, "dur": 0.974,
    "args": {
      "External id": 2470116,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509494.232, "dur": 2.160,
    "args": {
      "External id": 2470117,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509510.856, "dur": 0.834,
    "args": {
      "External id": 2470118,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509622.293, "dur": 2776.737,
    "args": {
      "External id": 2470119,"Record function id": 0, "Ev Idx": 3814
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.12)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509642.032, "dur": 1028.335,
    "args": {
      "External id": 2470120,"Record function id": 0, "Ev Idx": 3815
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509656.788, "dur": 320.248,
    "args": {
      "External id": 2470121,"Record function id": 0, "Ev Idx": 3816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509738.076, "dur": 6.368,
    "args": {
      "External id": 2470122,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509747.567, "dur": 1.472,
    "args": {
      "External id": 2470123,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509750.706, "dur": 1.291,
    "args": {
      "External id": 2470124,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509753.839, "dur": 1.183,
    "args": {
      "External id": 2470125,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509756.407, "dur": 0.869,
    "args": {
      "External id": 2470126,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509758.899, "dur": 0.846,
    "args": {
      "External id": 2470127,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509761.367, "dur": 1.305,
    "args": {
      "External id": 2470128,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509764.081, "dur": 0.647,
    "args": {
      "External id": 2470129,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509766.067, "dur": 2.617,
    "args": {
      "External id": 2470130,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509769.962, "dur": 1.138,
    "args": {
      "External id": 2470131,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509789.819, "dur": 153.797,
    "args": {
      "External id": 2470132,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509805.539, "dur": 133.672,
    "args": {
      "External id": 2470133,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509827.928, "dur": 13.428,
    "args": {
      "External id": 2470134,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509844.575, "dur": 65.870,
    "args": {
      "External id": 2470135,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509847.248, "dur": 62.815,
    "args": {
      "External id": 2470136,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509851.523, "dur": 5.620,
    "args": {
      "External id": 2470137,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657509859.173, "dur": 50.319,
    "args": {
      "External id": 2470138,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3833
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.11", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510106.366, "dur": 555.841,
    "args": {
      "External id": 2470139,"Record function id": 0, "Ev Idx": 3834
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510123.958, "dur": 525.156,
    "args": {
      "External id": 2470140,"Record function id": 0, "Ev Idx": 3835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510179.888, "dur": 5.468,
    "args": {
      "External id": 2470141,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510200.861, "dur": 30.033,
    "args": {
      "External id": 2470142,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510205.961, "dur": 1.617,
    "args": {
      "External id": 2470143,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510209.971, "dur": 0.514,
    "args": {
      "External id": 2470144,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510212.395, "dur": 0.363,
    "args": {
      "External id": 2470145,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510214.426, "dur": 2.884,
    "args": {
      "External id": 2470146,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510218.974, "dur": 0.433,
    "args": {
      "External id": 2470147,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510220.916, "dur": 0.566,
    "args": {
      "External id": 2470148,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510223.338, "dur": 0.399,
    "args": {
      "External id": 2470149,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510225.198, "dur": 0.498,
    "args": {
      "External id": 2470150,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510227.405, "dur": 0.462,
    "args": {
      "External id": 2470151,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510240.349, "dur": 33.528,
    "args": {
      "External id": 2470152,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510304.660, "dur": 100.136,
    "args": {
      "External id": 2470153,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510315.105, "dur": 3.388,
    "args": {
      "External id": 2470154,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510323.649, "dur": 11.102,
    "args": {
      "External id": 2470155,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510328.053, "dur": 6.272,
    "args": {
      "External id": 2470156,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510332.275, "dur": 0.904,
    "args": {
      "External id": 2470157,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510341.264, "dur": 28.452,
    "args": {
      "External id": 2470158,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510343.357, "dur": 2.297,
    "args": {
      "External id": 2470159,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510347.485, "dur": 0.317,
    "args": {
      "External id": 2470160,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510349.273, "dur": 0.526,
    "args": {
      "External id": 2470161,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510351.391, "dur": 0.402,
    "args": {
      "External id": 2470162,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510353.786, "dur": 0.476,
    "args": {
      "External id": 2470163,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510355.975, "dur": 0.390,
    "args": {
      "External id": 2470164,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510357.770, "dur": 0.438,
    "args": {
      "External id": 2470165,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510360.163, "dur": 0.425,
    "args": {
      "External id": 2470166,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510362.256, "dur": 2.440,
    "args": {
      "External id": 2470167,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510378.869, "dur": 18.650,
    "args": {
      "External id": 2470168,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510448.790, "dur": 131.873,
    "args": {
      "External id": 2470169,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510470.316, "dur": 106.190,
    "args": {
      "External id": 2470170,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3865, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510479.711, "dur": 92.120,
    "args": {
      "External id": 2470171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510595.945, "dur": 2.273,
    "args": {
      "External id": 2470172,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3867, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510677.638, "dur": 1701.766,
    "args": {
      "External id": 2470173,"Sequence number": 24740446, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3868
    }
  },
  {
    "ph": "f", "id": 202, "pid": 1336756, "tid": 1381189, "ts": 1514657510677.638,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510788.186, "dur": 106.497,
    "args": {
      "External id": 2470174,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657510934.281, "dur": 38.478,
    "args": {
      "External id": 2470175,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511035.791, "dur": 60.969,
    "args": {
      "External id": 2470176,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511110.670, "dur": 34.932,
    "args": {
      "External id": 2470177,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511152.785, "dur": 48.603,
    "args": {
      "External id": 2470178,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511209.633, "dur": 28.279,
    "args": {
      "External id": 2470179,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511248.380, "dur": 43.823,
    "args": {
      "External id": 2470180,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511316.245, "dur": 25.125,
    "args": {
      "External id": 2470181,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511359.794, "dur": 29.738,
    "args": {
      "External id": 2470182,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511408.381, "dur": 22.005,
    "args": {
      "External id": 2470183,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511444.348, "dur": 15.495,
    "args": {
      "External id": 2470184,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511470.590, "dur": 29.305,
    "args": {
      "External id": 2470185,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511503.246, "dur": 50.672,
    "args": {
      "External id": 2470186,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511584.457, "dur": 183.088,
    "args": {
      "External id": 2470187,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511666.121, "dur": 5.671,
    "args": {
      "External id": 2470188,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511674.088, "dur": 5.321,
    "args": {
      "External id": 2470189,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511800.600, "dur": 24.831,
    "args": {
      "External id": 2470190,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511836.831, "dur": 16.484,
    "args": {
      "External id": 2470191,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511862.508, "dur": 39.959,
    "args": {
      "External id": 2470192,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511911.430, "dur": 36.602,
    "args": {
      "External id": 2470193,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657511954.956, "dur": 20.964,
    "args": {
      "External id": 2470194,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512015.214, "dur": 40.934,
    "args": {
      "External id": 2470195,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512069.139, "dur": 21.651,
    "args": {
      "External id": 2470196,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512101.071, "dur": 51.231,
    "args": {
      "External id": 2470197,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512177.644, "dur": 29.437,
    "args": {
      "External id": 2470198,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512225.378, "dur": 28.250,
    "args": {
      "External id": 2470199,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512268.156, "dur": 18.311,
    "args": {
      "External id": 2470200,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512303.715, "dur": 15.823,
    "args": {
      "External id": 2470201,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512331.533, "dur": 16.875,
    "args": {
      "External id": 2470202,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512424.542, "dur": 17.889,
    "args": {
      "External id": 2470203,"Record function id": 0, "Ev Idx": 3898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512428.332, "dur": 13.160,
    "args": {
      "External id": 2470204,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512432.867, "dur": 7.713,
    "args": {
      "External id": 2470205,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512434.682, "dur": 5.755,
    "args": {
      "External id": 2470206,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512446.507, "dur": 5.855,
    "args": {
      "External id": 2470207,"Record function id": 0, "Ev Idx": 3902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512448.161, "dur": 3.751,
    "args": {
      "External id": 2470208,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512449.037, "dur": 2.312,
    "args": {
      "External id": 2470209,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512450.095, "dur": 1.158,
    "args": {
      "External id": 2470210,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512455.615, "dur": 5.121,
    "args": {
      "External id": 2470211,"Record function id": 0, "Ev Idx": 3906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512457.381, "dur": 2.930,
    "args": {
      "External id": 2470212,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512458.055, "dur": 1.761,
    "args": {
      "External id": 2470213,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512458.577, "dur": 1.150,
    "args": {
      "External id": 2470214,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512464.259, "dur": 4.641,
    "args": {
      "External id": 2470215,"Record function id": 0, "Ev Idx": 3910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512465.748, "dur": 2.761,
    "args": {
      "External id": 2470216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512466.677, "dur": 1.434,
    "args": {
      "External id": 2470217,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512467.128, "dur": 0.920,
    "args": {
      "External id": 2470218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512471.939, "dur": 7.464,
    "args": {
      "External id": 2470219,"Record function id": 0, "Ev Idx": 3914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512476.673, "dur": 2.328,
    "args": {
      "External id": 2470220,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512477.471, "dur": 1.133,
    "args": {
      "External id": 2470221,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512477.939, "dur": 0.600,
    "args": {
      "External id": 2470222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512482.584, "dur": 3.752,
    "args": {
      "External id": 2470223,"Record function id": 0, "Ev Idx": 3918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512483.766, "dur": 2.189,
    "args": {
      "External id": 2470224,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512484.391, "dur": 1.126,
    "args": {
      "External id": 2470225,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512484.829, "dur": 0.566,
    "args": {
      "External id": 2470226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512489.540, "dur": 3.744,
    "args": {
      "External id": 2470227,"Record function id": 0, "Ev Idx": 3922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512490.938, "dur": 1.964,
    "args": {
      "External id": 2470228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512491.540, "dur": 0.937,
    "args": {
      "External id": 2470229,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512491.828, "dur": 0.557,
    "args": {
      "External id": 2470230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512496.252, "dur": 5.407,
    "args": {
      "External id": 2470231,"Record function id": 0, "Ev Idx": 3926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512497.371, "dur": 3.910,
    "args": {
      "External id": 2470232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512497.804, "dur": 3.073,
    "args": {
      "External id": 2470233,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512498.184, "dur": 2.622,
    "args": {
      "External id": 2470234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512504.706, "dur": 41.734,
    "args": {
      "External id": 2470235,"Record function id": 0, "Ev Idx": 3930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512506.026, "dur": 39.320,
    "args": {
      "External id": 2470236,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512506.453, "dur": 37.732,
    "args": {
      "External id": 2470237,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512526.988, "dur": 16.743,
    "args": {
      "External id": 2470238,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512552.865, "dur": 35409.339,
    "args": {
      "External id": 2470239,"Record function id": 0, "Sequence number": 24740445, "Fwd thread id": 1, "Ev Idx": 3934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512554.823, "dur": 35398.826,
    "args": {
      "External id": 2470240,"Sequence number": 24740445, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3935
    }
  },
  {
    "ph": "f", "id": 203, "pid": 1336756, "tid": 1381189, "ts": 1514657512554.823,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.12)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512585.754, "dur": 37.362,
    "args": {
      "External id": 2470241,"Record function id": 0, "Ev Idx": 3936
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.12)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512630.703, "dur": 62.651,
    "args": {
      "External id": 2470242,"Record function id": 0, "Ev Idx": 3937
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.12)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512699.127, "dur": 35246.758,
    "args": {
      "External id": 2470243,"Record function id": 0, "Ev Idx": 3938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512785.637, "dur": 6.731,
    "args": {
      "External id": 2470244,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512802.369, "dur": 4.926,
    "args": {
      "External id": 2470245,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512822.636, "dur": 34310.174,
    "args": {
      "External id": 2470246,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512835.748, "dur": 34287.850,
    "args": {
      "External id": 2470247,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512873.179, "dur": 17.542,
    "args": {
      "External id": 2470248,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512897.079, "dur": 34184.346,
    "args": {
      "External id": 2470249,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512899.609, "dur": 34181.159,
    "args": {
      "External id": 2470250,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512904.124, "dur": 4.986,
    "args": {
      "External id": 2470251,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657512911.099, "dur": 34165.579,
    "args": {
      "External id": 2470252,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547218.528, "dur": 8.978,
    "args": {
      "External id": 2470253,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547221.722, "dur": 5.458,
    "args": {
      "External id": 2470254,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547259.202, "dur": 383.277,
    "args": {
      "External id": 2470255,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547284.597, "dur": 352.383,
    "args": {
      "External id": 2470256,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3951, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547295.364, "dur": 335.116,
    "args": {
      "External id": 2470257,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547663.514, "dur": 2.271,
    "args": {
      "External id": 2470258,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3953, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547726.441, "dur": 6.844,
    "args": {
      "External id": 2470259,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547786.329, "dur": 1.476,
    "args": {
      "External id": 2470260,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547804.116, "dur": 0.964,
    "args": {
      "External id": 2470261,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547817.907, "dur": 0.832,
    "args": {
      "External id": 2470262,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547830.675, "dur": 0.950,
    "args": {
      "External id": 2470263,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547845.477, "dur": 1.047,
    "args": {
      "External id": 2470264,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547858.373, "dur": 0.997,
    "args": {
      "External id": 2470265,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547871.204, "dur": 1.398,
    "args": {
      "External id": 2470266,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547884.239, "dur": 0.746,
    "args": {
      "External id": 2470267,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657547976.332, "dur": 2745.005,
    "args": {
      "External id": 2470268,"Record function id": 0, "Ev Idx": 3963
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.11)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548032.922, "dur": 1001.179,
    "args": {
      "External id": 2470269,"Record function id": 0, "Ev Idx": 3964
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548047.125, "dur": 316.704,
    "args": {
      "External id": 2470270,"Record function id": 0, "Ev Idx": 3965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548130.683, "dur": 4.709,
    "args": {
      "External id": 2470271,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548138.564, "dur": 1.669,
    "args": {
      "External id": 2470272,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548142.398, "dur": 1.136,
    "args": {
      "External id": 2470273,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548144.979, "dur": 1.079,
    "args": {
      "External id": 2470274,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548147.353, "dur": 1.321,
    "args": {
      "External id": 2470275,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548150.556, "dur": 0.859,
    "args": {
      "External id": 2470276,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548152.840, "dur": 0.967,
    "args": {
      "External id": 2470277,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548154.888, "dur": 3.086,
    "args": {
      "External id": 2470278,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548159.229, "dur": 0.978,
    "args": {
      "External id": 2470279,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548161.311, "dur": 1.182,
    "args": {
      "External id": 2470280,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548181.847, "dur": 148.331,
    "args": {
      "External id": 2470281,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548197.851, "dur": 128.295,
    "args": {
      "External id": 2470282,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548216.208, "dur": 12.089,
    "args": {
      "External id": 2470283,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548231.773, "dur": 65.355,
    "args": {
      "External id": 2470284,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548234.353, "dur": 62.466,
    "args": {
      "External id": 2470285,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548238.263, "dur": 5.441,
    "args": {
      "External id": 2470286,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548245.955, "dur": 50.273,
    "args": {
      "External id": 2470287,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3982
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.10", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548443.807, "dur": 580.373,
    "args": {
      "External id": 2470288,"Record function id": 0, "Ev Idx": 3983
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548460.902, "dur": 511.940,
    "args": {
      "External id": 2470289,"Record function id": 0, "Ev Idx": 3984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548515.350, "dur": 4.601,
    "args": {
      "External id": 2470290,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548550.930, "dur": 32.051,
    "args": {
      "External id": 2470291,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548555.715, "dur": 1.694,
    "args": {
      "External id": 2470292,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548559.813, "dur": 0.669,
    "args": {
      "External id": 2470293,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548562.315, "dur": 2.553,
    "args": {
      "External id": 2470294,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548566.853, "dur": 0.693,
    "args": {
      "External id": 2470295,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548568.954, "dur": 0.590,
    "args": {
      "External id": 2470296,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548571.040, "dur": 0.391,
    "args": {
      "External id": 2470297,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548573.069, "dur": 0.376,
    "args": {
      "External id": 2470298,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548575.144, "dur": 0.598,
    "args": {
      "External id": 2470299,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548577.488, "dur": 0.346,
    "args": {
      "External id": 2470300,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548593.393, "dur": 33.118,
    "args": {
      "External id": 2470301,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548658.441, "dur": 101.929,
    "args": {
      "External id": 2470302,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548669.395, "dur": 3.728,
    "args": {
      "External id": 2470303,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548678.030, "dur": 11.785,
    "args": {
      "External id": 2470304,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548682.258, "dur": 7.150,
    "args": {
      "External id": 2470305,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548685.631, "dur": 2.412,
    "args": {
      "External id": 2470306,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548696.413, "dur": 26.307,
    "args": {
      "External id": 2470307,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548698.443, "dur": 0.568,
    "args": {
      "External id": 2470308,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548701.069, "dur": 0.354,
    "args": {
      "External id": 2470309,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548702.908, "dur": 0.541,
    "args": {
      "External id": 2470310,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548705.443, "dur": 0.372,
    "args": {
      "External id": 2470311,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548707.595, "dur": 0.354,
    "args": {
      "External id": 2470312,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548709.687, "dur": 0.381,
    "args": {
      "External id": 2470313,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548711.481, "dur": 0.448,
    "args": {
      "External id": 2470314,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548713.758, "dur": 2.104,
    "args": {
      "External id": 2470315,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548717.401, "dur": 0.363,
    "args": {
      "External id": 2470316,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548731.632, "dur": 21.013,
    "args": {
      "External id": 2470317,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548804.726, "dur": 105.493,
    "args": {
      "External id": 2470318,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548825.552, "dur": 81.174,
    "args": {
      "External id": 2470319,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4014, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548834.721, "dur": 68.121,
    "args": {
      "External id": 2470320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657548923.551, "dur": 1.562,
    "args": {
      "External id": 2470321,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4016, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549041.324, "dur": 1656.142,
    "args": {
      "External id": 2470322,"Sequence number": 24740444, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4017
    }
  },
  {
    "ph": "f", "id": 204, "pid": 1336756, "tid": 1381189, "ts": 1514657549041.324,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549157.262, "dur": 113.767,
    "args": {
      "External id": 2470323,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549307.451, "dur": 40.240,
    "args": {
      "External id": 2470324,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549367.305, "dur": 50.399,
    "args": {
      "External id": 2470325,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549428.172, "dur": 32.759,
    "args": {
      "External id": 2470326,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549468.300, "dur": 46.062,
    "args": {
      "External id": 2470327,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549521.948, "dur": 49.464,
    "args": {
      "External id": 2470328,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549584.292, "dur": 46.724,
    "args": {
      "External id": 2470329,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549653.790, "dur": 23.695,
    "args": {
      "External id": 2470330,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549695.137, "dur": 30.625,
    "args": {
      "External id": 2470331,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549744.043, "dur": 20.211,
    "args": {
      "External id": 2470332,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549777.210, "dur": 16.092,
    "args": {
      "External id": 2470333,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549803.568, "dur": 28.984,
    "args": {
      "External id": 2470334,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549836.049, "dur": 33.663,
    "args": {
      "External id": 2470335,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549896.129, "dur": 224.036,
    "args": {
      "External id": 2470336,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657549977.728, "dur": 42.586,
    "args": {
      "External id": 2470337,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550023.832, "dur": 2.631,
    "args": {
      "External id": 2470338,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550152.373, "dur": 25.216,
    "args": {
      "External id": 2470339,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550188.236, "dur": 15.719,
    "args": {
      "External id": 2470340,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550215.221, "dur": 42.077,
    "args": {
      "External id": 2470341,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550264.535, "dur": 36.816,
    "args": {
      "External id": 2470342,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550308.293, "dur": 25.671,
    "args": {
      "External id": 2470343,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550339.664, "dur": 34.833,
    "args": {
      "External id": 2470344,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550383.185, "dur": 22.347,
    "args": {
      "External id": 2470345,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550412.681, "dur": 32.157,
    "args": {
      "External id": 2470346,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550476.539, "dur": 28.758,
    "args": {
      "External id": 2470347,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550525.467, "dur": 40.477,
    "args": {
      "External id": 2470348,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550581.652, "dur": 21.450,
    "args": {
      "External id": 2470349,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550618.323, "dur": 16.006,
    "args": {
      "External id": 2470350,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550650.461, "dur": 17.287,
    "args": {
      "External id": 2470351,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550744.149, "dur": 19.433,
    "args": {
      "External id": 2470352,"Record function id": 0, "Ev Idx": 4047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550747.672, "dur": 14.962,
    "args": {
      "External id": 2470353,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550752.034, "dur": 9.672,
    "args": {
      "External id": 2470354,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550753.869, "dur": 7.699,
    "args": {
      "External id": 2470355,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550767.501, "dur": 5.273,
    "args": {
      "External id": 2470356,"Record function id": 0, "Ev Idx": 4051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550768.814, "dur": 3.513,
    "args": {
      "External id": 2470357,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550769.483, "dur": 2.376,
    "args": {
      "External id": 2470358,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550770.487, "dur": 1.281,
    "args": {
      "External id": 2470359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550776.152, "dur": 4.254,
    "args": {
      "External id": 2470360,"Record function id": 0, "Ev Idx": 4055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550777.407, "dur": 2.577,
    "args": {
      "External id": 2470361,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550777.927, "dur": 1.607,
    "args": {
      "External id": 2470362,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550778.603, "dur": 0.863,
    "args": {
      "External id": 2470363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550783.510, "dur": 4.323,
    "args": {
      "External id": 2470364,"Record function id": 0, "Ev Idx": 4059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550784.856, "dur": 2.576,
    "args": {
      "External id": 2470365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550785.809, "dur": 1.231,
    "args": {
      "External id": 2470366,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550786.352, "dur": 0.609,
    "args": {
      "External id": 2470367,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550790.900, "dur": 3.533,
    "args": {
      "External id": 2470368,"Record function id": 0, "Ev Idx": 4063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550791.868, "dur": 2.115,
    "args": {
      "External id": 2470369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550792.419, "dur": 1.163,
    "args": {
      "External id": 2470370,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550792.953, "dur": 0.564,
    "args": {
      "External id": 2470371,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550797.466, "dur": 3.473,
    "args": {
      "External id": 2470372,"Record function id": 0, "Ev Idx": 4067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550798.399, "dur": 2.107,
    "args": {
      "External id": 2470373,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550798.916, "dur": 1.183,
    "args": {
      "External id": 2470374,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550799.346, "dur": 0.635,
    "args": {
      "External id": 2470375,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550804.125, "dur": 5.314,
    "args": {
      "External id": 2470376,"Record function id": 0, "Ev Idx": 4071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550805.166, "dur": 3.874,
    "args": {
      "External id": 2470377,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550805.630, "dur": 3.015,
    "args": {
      "External id": 2470378,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550805.904, "dur": 2.670,
    "args": {
      "External id": 2470379,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550812.447, "dur": 3.498,
    "args": {
      "External id": 2470380,"Record function id": 0, "Ev Idx": 4075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550813.427, "dur": 2.105,
    "args": {
      "External id": 2470381,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550813.943, "dur": 1.156,
    "args": {
      "External id": 2470382,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550814.195, "dur": 0.824,
    "args": {
      "External id": 2470383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550818.875, "dur": 3.392,
    "args": {
      "External id": 2470384,"Record function id": 0, "Ev Idx": 4079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550819.922, "dur": 1.934,
    "args": {
      "External id": 2470385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550820.370, "dur": 1.070,
    "args": {
      "External id": 2470386,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550820.831, "dur": 0.521,
    "args": {
      "External id": 2470387,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550825.598, "dur": 35689.483,
    "args": {
      "External id": 2470388,"Record function id": 0, "Sequence number": 24740443, "Fwd thread id": 1, "Ev Idx": 4083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550826.991, "dur": 35679.752,
    "args": {
      "External id": 2470389,"Sequence number": 24740443, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4084
    }
  },
  {
    "ph": "f", "id": 205, "pid": 1336756, "tid": 1381189, "ts": 1514657550826.991,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.11)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550855.474, "dur": 38.969,
    "args": {
      "External id": 2470390,"Record function id": 0, "Ev Idx": 4085
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.11)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550902.398, "dur": 62.151,
    "args": {
      "External id": 2470391,"Record function id": 0, "Ev Idx": 4086
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.11)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657550971.491, "dur": 35527.770,
    "args": {
      "External id": 2470392,"Record function id": 0, "Ev Idx": 4087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551104.258, "dur": 6.839,
    "args": {
      "External id": 2470393,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551121.933, "dur": 5.049,
    "args": {
      "External id": 2470394,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551143.554, "dur": 34540.706,
    "args": {
      "External id": 2470395,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551156.583, "dur": 34518.557,
    "args": {
      "External id": 2470396,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551214.745, "dur": 13.973,
    "args": {
      "External id": 2470397,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551234.974, "dur": 34400.586,
    "args": {
      "External id": 2470398,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551237.703, "dur": 34397.041,
    "args": {
      "External id": 2470399,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551241.566, "dur": 4.630,
    "args": {
      "External id": 2470400,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657551267.539, "dur": 34363.713,
    "args": {
      "External id": 2470401,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657585776.322, "dur": 8.728,
    "args": {
      "External id": 2470402,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657585779.567, "dur": 5.163,
    "args": {
      "External id": 2470403,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657585816.682, "dur": 381.643,
    "args": {
      "External id": 2470404,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657585842.233, "dur": 350.737,
    "args": {
      "External id": 2470405,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4100, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657585852.945, "dur": 334.103,
    "args": {
      "External id": 2470406,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586219.255, "dur": 2.269,
    "args": {
      "External id": 2470407,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4102, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586282.265, "dur": 9.122,
    "args": {
      "External id": 2470408,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586337.352, "dur": 1.504,
    "args": {
      "External id": 2470409,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586354.553, "dur": 1.221,
    "args": {
      "External id": 2470410,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586368.815, "dur": 1.248,
    "args": {
      "External id": 2470411,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586382.374, "dur": 3.275,
    "args": {
      "External id": 2470412,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586396.234, "dur": 1.262,
    "args": {
      "External id": 2470413,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586412.795, "dur": 1.117,
    "args": {
      "External id": 2470414,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586425.680, "dur": 0.895,
    "args": {
      "External id": 2470415,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586436.102, "dur": 3.275,
    "args": {
      "External id": 2470416,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586546.518, "dur": 2787.917,
    "args": {
      "External id": 2470417,"Record function id": 0, "Ev Idx": 4112
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.10)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586570.138, "dur": 1023.126,
    "args": {
      "External id": 2470418,"Record function id": 0, "Ev Idx": 4113
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586585.001, "dur": 317.539,
    "args": {
      "External id": 2470419,"Record function id": 0, "Ev Idx": 4114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586665.809, "dur": 4.764,
    "args": {
      "External id": 2470420,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586673.571, "dur": 1.113,
    "args": {
      "External id": 2470421,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586676.533, "dur": 1.056,
    "args": {
      "External id": 2470422,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586682.429, "dur": 1.173,
    "args": {
      "External id": 2470423,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586684.967, "dur": 0.893,
    "args": {
      "External id": 2470424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586687.323, "dur": 0.905,
    "args": {
      "External id": 2470425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586689.836, "dur": 2.498,
    "args": {
      "External id": 2470426,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586693.626, "dur": 1.090,
    "args": {
      "External id": 2470427,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586696.040, "dur": 0.980,
    "args": {
      "External id": 2470428,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586698.256, "dur": 0.850,
    "args": {
      "External id": 2470429,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586717.692, "dur": 151.826,
    "args": {
      "External id": 2470430,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586732.576, "dur": 132.463,
    "args": {
      "External id": 2470431,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586754.100, "dur": 12.935,
    "args": {
      "External id": 2470432,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586770.042, "dur": 65.415,
    "args": {
      "External id": 2470433,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586772.635, "dur": 62.490,
    "args": {
      "External id": 2470434,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586776.913, "dur": 5.015,
    "args": {
      "External id": 2470435,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586783.621, "dur": 50.643,
    "args": {
      "External id": 2470436,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4131
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.9", "pid": 1336756, "tid": 1381189,
    "ts": 1514657586979.598, "dur": 605.373,
    "args": {
      "External id": 2470437,"Record function id": 0, "Ev Idx": 4132
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587038.140, "dur": 533.491,
    "args": {
      "External id": 2470438,"Record function id": 0, "Ev Idx": 4133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587097.604, "dur": 6.440,
    "args": {
      "External id": 2470439,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587119.604, "dur": 33.610,
    "args": {
      "External id": 2470440,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587124.510, "dur": 1.714,
    "args": {
      "External id": 2470441,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587128.159, "dur": 2.484,
    "args": {
      "External id": 2470442,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587132.216, "dur": 0.583,
    "args": {
      "External id": 2470443,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587134.404, "dur": 0.425,
    "args": {
      "External id": 2470444,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587136.424, "dur": 0.345,
    "args": {
      "External id": 2470445,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587138.232, "dur": 0.370,
    "args": {
      "External id": 2470446,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587144.809, "dur": 0.452,
    "args": {
      "External id": 2470447,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587146.653, "dur": 0.384,
    "args": {
      "External id": 2470448,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587148.520, "dur": 0.367,
    "args": {
      "External id": 2470449,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587163.015, "dur": 35.144,
    "args": {
      "External id": 2470450,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587228.711, "dur": 103.681,
    "args": {
      "External id": 2470451,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587239.158, "dur": 5.162,
    "args": {
      "External id": 2470452,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587249.674, "dur": 10.391,
    "args": {
      "External id": 2470453,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587253.999, "dur": 5.630,
    "args": {
      "External id": 2470454,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587257.811, "dur": 0.452,
    "args": {
      "External id": 2470455,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587270.012, "dur": 25.045,
    "args": {
      "External id": 2470456,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587272.308, "dur": 0.477,
    "args": {
      "External id": 2470457,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587274.377, "dur": 0.370,
    "args": {
      "External id": 2470458,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587276.370, "dur": 0.480,
    "args": {
      "External id": 2470459,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587278.106, "dur": 0.406,
    "args": {
      "External id": 2470460,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587280.018, "dur": 0.603,
    "args": {
      "External id": 2470461,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587281.829, "dur": 0.365,
    "args": {
      "External id": 2470462,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587283.780, "dur": 2.431,
    "args": {
      "External id": 2470463,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587288.028, "dur": 0.656,
    "args": {
      "External id": 2470464,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587290.428, "dur": 0.477,
    "args": {
      "External id": 2470465,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587304.818, "dur": 20.420,
    "args": {
      "External id": 2470466,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587377.980, "dur": 110.451,
    "args": {
      "External id": 2470467,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587398.999, "dur": 86.074,
    "args": {
      "External id": 2470468,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4163, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587408.306, "dur": 72.751,
    "args": {
      "External id": 2470469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587504.151, "dur": 1.760,
    "args": {
      "External id": 2470470,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4165, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587600.017, "dur": 1715.025,
    "args": {
      "External id": 2470471,"Sequence number": 24740442, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4166
    }
  },
  {
    "ph": "f", "id": 206, "pid": 1336756, "tid": 1381189, "ts": 1514657587600.017,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587707.918, "dur": 107.561,
    "args": {
      "External id": 2470472,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587852.195, "dur": 40.720,
    "args": {
      "External id": 2470473,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587912.658, "dur": 52.658,
    "args": {
      "External id": 2470474,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657587975.063, "dur": 76.283,
    "args": {
      "External id": 2470475,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588061.598, "dur": 48.732,
    "args": {
      "External id": 2470476,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588120.480, "dur": 29.754,
    "args": {
      "External id": 2470477,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588157.640, "dur": 43.734,
    "args": {
      "External id": 2470478,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588224.228, "dur": 24.975,
    "args": {
      "External id": 2470479,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588283.285, "dur": 32.383,
    "args": {
      "External id": 2470480,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588339.507, "dur": 20.633,
    "args": {
      "External id": 2470481,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588372.780, "dur": 16.763,
    "args": {
      "External id": 2470482,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588399.293, "dur": 31.174,
    "args": {
      "External id": 2470483,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588433.767, "dur": 35.070,
    "args": {
      "External id": 2470484,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588496.402, "dur": 198.531,
    "args": {
      "External id": 2470485,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588594.769, "dur": 6.443,
    "args": {
      "External id": 2470486,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588603.402, "dur": 3.905,
    "args": {
      "External id": 2470487,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588728.353, "dur": 26.045,
    "args": {
      "External id": 2470488,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588766.078, "dur": 16.045,
    "args": {
      "External id": 2470489,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588791.251, "dur": 39.834,
    "args": {
      "External id": 2470490,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588837.775, "dur": 39.267,
    "args": {
      "External id": 2470491,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588883.630, "dur": 25.693,
    "args": {
      "External id": 2470492,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588916.703, "dur": 31.869,
    "args": {
      "External id": 2470493,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657588955.089, "dur": 80.216,
    "args": {
      "External id": 2470494,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589054.807, "dur": 42.787,
    "args": {
      "External id": 2470495,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589117.269, "dur": 26.460,
    "args": {
      "External id": 2470496,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589160.528, "dur": 27.256,
    "args": {
      "External id": 2470497,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589201.513, "dur": 18.602,
    "args": {
      "External id": 2470498,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589234.702, "dur": 20.261,
    "args": {
      "External id": 2470499,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589268.926, "dur": 17.183,
    "args": {
      "External id": 2470500,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589355.895, "dur": 14.850,
    "args": {
      "External id": 2470501,"Record function id": 0, "Ev Idx": 4196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589358.983, "dur": 10.844,
    "args": {
      "External id": 2470502,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589363.137, "dur": 5.790,
    "args": {
      "External id": 2470503,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589364.765, "dur": 4.042,
    "args": {
      "External id": 2470504,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589374.479, "dur": 5.101,
    "args": {
      "External id": 2470505,"Record function id": 0, "Ev Idx": 4200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589375.999, "dur": 3.088,
    "args": {
      "External id": 2470506,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589376.687, "dur": 1.959,
    "args": {
      "External id": 2470507,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589377.542, "dur": 1.024,
    "args": {
      "External id": 2470508,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589382.894, "dur": 5.253,
    "args": {
      "External id": 2470509,"Record function id": 0, "Ev Idx": 4204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589384.341, "dur": 3.379,
    "args": {
      "External id": 2470510,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589385.206, "dur": 2.053,
    "args": {
      "External id": 2470511,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589386.036, "dur": 1.133,
    "args": {
      "External id": 2470512,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589391.323, "dur": 6.115,
    "args": {
      "External id": 2470513,"Record function id": 0, "Ev Idx": 4208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589392.403, "dur": 4.597,
    "args": {
      "External id": 2470514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589392.920, "dur": 3.685,
    "args": {
      "External id": 2470515,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589393.466, "dur": 3.058,
    "args": {
      "External id": 2470516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589400.469, "dur": 3.970,
    "args": {
      "External id": 2470517,"Record function id": 0, "Ev Idx": 4212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589401.643, "dur": 2.381,
    "args": {
      "External id": 2470518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589402.147, "dur": 1.478,
    "args": {
      "External id": 2470519,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589402.463, "dur": 1.077,
    "args": {
      "External id": 2470520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589407.548, "dur": 4.052,
    "args": {
      "External id": 2470521,"Record function id": 0, "Ev Idx": 4216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589408.638, "dur": 2.479,
    "args": {
      "External id": 2470522,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589409.365, "dur": 1.315,
    "args": {
      "External id": 2470523,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589409.821, "dur": 0.741,
    "args": {
      "External id": 2470524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589414.852, "dur": 3.968,
    "args": {
      "External id": 2470525,"Record function id": 0, "Ev Idx": 4220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589416.244, "dur": 2.153,
    "args": {
      "External id": 2470526,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589416.896, "dur": 1.096,
    "args": {
      "External id": 2470527,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589417.258, "dur": 0.650,
    "args": {
      "External id": 2470528,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589421.914, "dur": 3.426,
    "args": {
      "External id": 2470529,"Record function id": 0, "Ev Idx": 4224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589423.034, "dur": 1.889,
    "args": {
      "External id": 2470530,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589423.500, "dur": 1.007,
    "args": {
      "External id": 2470531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589423.780, "dur": 0.654,
    "args": {
      "External id": 2470532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589428.311, "dur": 3.883,
    "args": {
      "External id": 2470533,"Record function id": 0, "Ev Idx": 4228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589429.676, "dur": 2.072,
    "args": {
      "External id": 2470534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589430.155, "dur": 1.199,
    "args": {
      "External id": 2470535,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589430.647, "dur": 0.620,
    "args": {
      "External id": 2470536,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589435.801, "dur": 35526.287,
    "args": {
      "External id": 2470537,"Record function id": 0, "Sequence number": 24740441, "Fwd thread id": 1, "Ev Idx": 4232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589437.279, "dur": 35516.252,
    "args": {
      "External id": 2470538,"Sequence number": 24740441, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4233
    }
  },
  {
    "ph": "f", "id": 207, "pid": 1336756, "tid": 1381189, "ts": 1514657589437.279,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.10)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589464.804, "dur": 36.338,
    "args": {
      "External id": 2470539,"Record function id": 0, "Ev Idx": 4234
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.10)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589508.541, "dur": 83.621,
    "args": {
      "External id": 2470540,"Record function id": 0, "Ev Idx": 4235
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.10)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589600.457, "dur": 35345.512,
    "args": {
      "External id": 2470541,"Record function id": 0, "Ev Idx": 4236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589691.322, "dur": 6.761,
    "args": {
      "External id": 2470542,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589708.376, "dur": 5.025,
    "args": {
      "External id": 2470543,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589730.910, "dur": 34367.045,
    "args": {
      "External id": 2470544,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589744.047, "dur": 34344.845,
    "args": {
      "External id": 2470545,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589804.223, "dur": 15.629,
    "args": {
      "External id": 2470546,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589826.409, "dur": 34222.742,
    "args": {
      "External id": 2470547,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589828.995, "dur": 34219.451,
    "args": {
      "External id": 2470548,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589832.678, "dur": 5.038,
    "args": {
      "External id": 2470549,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657589839.662, "dur": 34204.909,
    "args": {
      "External id": 2470550,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624187.298, "dur": 8.950,
    "args": {
      "External id": 2470551,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624190.512, "dur": 5.451,
    "args": {
      "External id": 2470552,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624227.817, "dur": 419.132,
    "args": {
      "External id": 2470553,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624253.608, "dur": 387.803,
    "args": {
      "External id": 2470554,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4249, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624264.869, "dur": 370.764,
    "args": {
      "External id": 2470555,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624669.243, "dur": 2.558,
    "args": {
      "External id": 2470556,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4251, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624735.446, "dur": 6.985,
    "args": {
      "External id": 2470557,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624788.779, "dur": 1.584,
    "args": {
      "External id": 2470558,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624808.178, "dur": 1.098,
    "args": {
      "External id": 2470559,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624823.951, "dur": 1.197,
    "args": {
      "External id": 2470560,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624836.732, "dur": 1.058,
    "args": {
      "External id": 2470561,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624848.229, "dur": 1.026,
    "args": {
      "External id": 2470562,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624859.720, "dur": 0.834,
    "args": {
      "External id": 2470563,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624873.613, "dur": 0.858,
    "args": {
      "External id": 2470564,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624884.840, "dur": 0.865,
    "args": {
      "External id": 2470565,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657624975.811, "dur": 2819.695,
    "args": {
      "External id": 2470566,"Record function id": 0, "Ev Idx": 4261
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.9)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625033.811, "dur": 1037.902,
    "args": {
      "External id": 2470567,"Record function id": 0, "Ev Idx": 4262
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625048.990, "dur": 331.270,
    "args": {
      "External id": 2470568,"Record function id": 0, "Ev Idx": 4263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625131.615, "dur": 5.172,
    "args": {
      "External id": 2470569,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625140.107, "dur": 1.476,
    "args": {
      "External id": 2470570,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625143.436, "dur": 1.497,
    "args": {
      "External id": 2470571,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625146.410, "dur": 2.982,
    "args": {
      "External id": 2470572,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625150.720, "dur": 1.024,
    "args": {
      "External id": 2470573,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625153.599, "dur": 1.037,
    "args": {
      "External id": 2470574,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625155.956, "dur": 0.961,
    "args": {
      "External id": 2470575,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625158.214, "dur": 1.151,
    "args": {
      "External id": 2470576,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625160.956, "dur": 0.740,
    "args": {
      "External id": 2470577,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625163.101, "dur": 0.941,
    "args": {
      "External id": 2470578,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625182.836, "dur": 162.078,
    "args": {
      "External id": 2470579,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625198.623, "dur": 141.912,
    "args": {
      "External id": 2470580,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625222.369, "dur": 12.920,
    "args": {
      "External id": 2470581,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625238.667, "dur": 71.924,
    "args": {
      "External id": 2470582,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625241.515, "dur": 68.716,
    "args": {
      "External id": 2470583,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625245.551, "dur": 8.490,
    "args": {
      "External id": 2470584,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625257.956, "dur": 51.278,
    "args": {
      "External id": 2470585,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4280
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.8", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625460.623, "dur": 603.326,
    "args": {
      "External id": 2470586,"Record function id": 0, "Ev Idx": 4281
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625478.453, "dur": 572.671,
    "args": {
      "External id": 2470587,"Record function id": 0, "Ev Idx": 4282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625554.856, "dur": 6.221,
    "args": {
      "External id": 2470588,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625577.745, "dur": 34.359,
    "args": {
      "External id": 2470589,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625586.238, "dur": 1.663,
    "args": {
      "External id": 2470590,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625590.344, "dur": 0.722,
    "args": {
      "External id": 2470591,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625592.694, "dur": 0.642,
    "args": {
      "External id": 2470592,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625595.029, "dur": 0.881,
    "args": {
      "External id": 2470593,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625597.515, "dur": 0.550,
    "args": {
      "External id": 2470594,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625599.624, "dur": 0.489,
    "args": {
      "External id": 2470595,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625601.584, "dur": 2.094,
    "args": {
      "External id": 2470596,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625605.529, "dur": 0.455,
    "args": {
      "External id": 2470597,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625607.525, "dur": 0.368,
    "args": {
      "External id": 2470598,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625622.095, "dur": 33.587,
    "args": {
      "External id": 2470599,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625688.486, "dur": 98.942,
    "args": {
      "External id": 2470600,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625699.239, "dur": 2.647,
    "args": {
      "External id": 2470601,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625707.777, "dur": 10.226,
    "args": {
      "External id": 2470602,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625712.044, "dur": 5.554,
    "args": {
      "External id": 2470603,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625715.756, "dur": 0.503,
    "args": {
      "External id": 2470604,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625725.188, "dur": 24.200,
    "args": {
      "External id": 2470605,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625727.300, "dur": 0.820,
    "args": {
      "External id": 2470606,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625729.966, "dur": 0.511,
    "args": {
      "External id": 2470607,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625732.050, "dur": 0.500,
    "args": {
      "External id": 2470608,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625733.857, "dur": 2.605,
    "args": {
      "External id": 2470609,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625738.030, "dur": 0.438,
    "args": {
      "External id": 2470610,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625740.086, "dur": 0.416,
    "args": {
      "External id": 2470611,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625742.143, "dur": 0.417,
    "args": {
      "External id": 2470612,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625744.231, "dur": 0.532,
    "args": {
      "External id": 2470613,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625746.154, "dur": 0.391,
    "args": {
      "External id": 2470614,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625759.410, "dur": 20.593,
    "args": {
      "External id": 2470615,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625833.092, "dur": 111.597,
    "args": {
      "External id": 2470616,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625854.731, "dur": 86.496,
    "args": {
      "External id": 2470617,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4312, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625864.558, "dur": 72.476,
    "args": {
      "External id": 2470618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657625957.358, "dur": 1.980,
    "args": {
      "External id": 2470619,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4314, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626079.353, "dur": 1694.688,
    "args": {
      "External id": 2470620,"Sequence number": 24740440, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4315
    }
  },
  {
    "ph": "f", "id": 208, "pid": 1336756, "tid": 1381189, "ts": 1514657626079.353,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626191.486, "dur": 110.103,
    "args": {
      "External id": 2470621,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626341.765, "dur": 41.929,
    "args": {
      "External id": 2470622,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626400.464, "dur": 51.479,
    "args": {
      "External id": 2470623,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626462.454, "dur": 33.670,
    "args": {
      "External id": 2470624,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626505.480, "dur": 63.185,
    "args": {
      "External id": 2470625,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626580.120, "dur": 33.045,
    "args": {
      "External id": 2470626,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626620.998, "dur": 43.338,
    "args": {
      "External id": 2470627,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626687.884, "dur": 25.772,
    "args": {
      "External id": 2470628,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626733.388, "dur": 31.886,
    "args": {
      "External id": 2470629,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626783.072, "dur": 19.634,
    "args": {
      "External id": 2470630,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626814.837, "dur": 16.528,
    "args": {
      "External id": 2470631,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626840.574, "dur": 29.507,
    "args": {
      "External id": 2470632,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626872.988, "dur": 33.350,
    "args": {
      "External id": 2470633,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657626935.200, "dur": 221.572,
    "args": {
      "External id": 2470634,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627053.684, "dur": 6.815,
    "args": {
      "External id": 2470635,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627062.880, "dur": 1.934,
    "args": {
      "External id": 2470636,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627192.627, "dur": 25.484,
    "args": {
      "External id": 2470637,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627229.244, "dur": 16.251,
    "args": {
      "External id": 2470638,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627257.435, "dur": 41.494,
    "args": {
      "External id": 2470639,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627306.433, "dur": 41.545,
    "args": {
      "External id": 2470640,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627356.976, "dur": 22.090,
    "args": {
      "External id": 2470641,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627398.562, "dur": 33.820,
    "args": {
      "External id": 2470642,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627439.732, "dur": 37.270,
    "args": {
      "External id": 2470643,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627489.095, "dur": 37.147,
    "args": {
      "External id": 2470644,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627564.266, "dur": 30.574,
    "args": {
      "External id": 2470645,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627611.640, "dur": 25.091,
    "args": {
      "External id": 2470646,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627655.849, "dur": 18.349,
    "args": {
      "External id": 2470647,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627692.314, "dur": 19.814,
    "args": {
      "External id": 2470648,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627724.645, "dur": 17.957,
    "args": {
      "External id": 2470649,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627816.919, "dur": 19.308,
    "args": {
      "External id": 2470650,"Record function id": 0, "Ev Idx": 4345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627820.424, "dur": 14.608,
    "args": {
      "External id": 2470651,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627827.981, "dur": 6.289,
    "args": {
      "External id": 2470652,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627829.928, "dur": 4.194,
    "args": {
      "External id": 2470653,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627840.091, "dur": 5.278,
    "args": {
      "External id": 2470654,"Record function id": 0, "Ev Idx": 4349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627841.539, "dur": 3.377,
    "args": {
      "External id": 2470655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627842.591, "dur": 1.829,
    "args": {
      "External id": 2470656,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627843.228, "dur": 1.095,
    "args": {
      "External id": 2470657,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627848.568, "dur": 4.947,
    "args": {
      "External id": 2470658,"Record function id": 0, "Ev Idx": 4353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627850.100, "dur": 3.003,
    "args": {
      "External id": 2470659,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627850.764, "dur": 1.904,
    "args": {
      "External id": 2470660,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627851.420, "dur": 1.181,
    "args": {
      "External id": 2470661,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627856.654, "dur": 4.283,
    "args": {
      "External id": 2470662,"Record function id": 0, "Ev Idx": 4357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627858.080, "dur": 2.456,
    "args": {
      "External id": 2470663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627858.738, "dur": 1.368,
    "args": {
      "External id": 2470664,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627859.250, "dur": 0.789,
    "args": {
      "External id": 2470665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627864.014, "dur": 5.367,
    "args": {
      "External id": 2470666,"Record function id": 0, "Ev Idx": 4361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627865.217, "dur": 3.729,
    "args": {
      "External id": 2470667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627865.763, "dur": 2.764,
    "args": {
      "External id": 2470668,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627866.019, "dur": 2.446,
    "args": {
      "External id": 2470669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627872.501, "dur": 4.384,
    "args": {
      "External id": 2470670,"Record function id": 0, "Ev Idx": 4365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627873.784, "dur": 2.655,
    "args": {
      "External id": 2470671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627874.450, "dur": 1.558,
    "args": {
      "External id": 2470672,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627874.945, "dur": 0.913,
    "args": {
      "External id": 2470673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627880.113, "dur": 7.703,
    "args": {
      "External id": 2470674,"Record function id": 0, "Ev Idx": 4369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627881.412, "dur": 5.985,
    "args": {
      "External id": 2470675,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627886.003, "dur": 0.974,
    "args": {
      "External id": 2470676,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627886.283, "dur": 0.634,
    "args": {
      "External id": 2470677,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627890.812, "dur": 3.879,
    "args": {
      "External id": 2470678,"Record function id": 0, "Ev Idx": 4373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627892.051, "dur": 2.212,
    "args": {
      "External id": 2470679,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627892.511, "dur": 1.322,
    "args": {
      "External id": 2470680,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627892.896, "dur": 0.845,
    "args": {
      "External id": 2470681,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627897.675, "dur": 4.567,
    "args": {
      "External id": 2470682,"Record function id": 0, "Ev Idx": 4377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627899.017, "dur": 2.796,
    "args": {
      "External id": 2470683,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627899.568, "dur": 1.794,
    "args": {
      "External id": 2470684,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627900.428, "dur": 0.805,
    "args": {
      "External id": 2470685,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627905.923, "dur": 35037.853,
    "args": {
      "External id": 2470686,"Record function id": 0, "Sequence number": 24740439, "Fwd thread id": 1, "Ev Idx": 4381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627907.636, "dur": 35027.036,
    "args": {
      "External id": 2470687,"Sequence number": 24740439, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4382
    }
  },
  {
    "ph": "f", "id": 209, "pid": 1336756, "tid": 1381189, "ts": 1514657627907.636,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.9)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627934.429, "dur": 39.869,
    "args": {
      "External id": 2470688,"Record function id": 0, "Ev Idx": 4383
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.9)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657627981.571, "dur": 106.736,
    "args": {
      "External id": 2470689,"Record function id": 0, "Ev Idx": 4384
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.9)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628097.046, "dur": 34829.820,
    "args": {
      "External id": 2470690,"Record function id": 0, "Ev Idx": 4385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628188.233, "dur": 7.093,
    "args": {
      "External id": 2470691,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628206.055, "dur": 5.220,
    "args": {
      "External id": 2470692,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628225.706, "dur": 33889.780,
    "args": {
      "External id": 2470693,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628239.096, "dur": 33867.430,
    "args": {
      "External id": 2470694,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628289.076, "dur": 14.038,
    "args": {
      "External id": 2470695,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628309.360, "dur": 33753.988,
    "args": {
      "External id": 2470696,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628311.871, "dur": 33750.859,
    "args": {
      "External id": 2470697,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628315.867, "dur": 7.205,
    "args": {
      "External id": 2470698,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657628324.977, "dur": 33733.759,
    "args": {
      "External id": 2470699,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662203.939, "dur": 9.082,
    "args": {
      "External id": 2470700,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662207.468, "dur": 5.247,
    "args": {
      "External id": 2470701,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662243.188, "dur": 390.729,
    "args": {
      "External id": 2470702,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662268.886, "dur": 359.579,
    "args": {
      "External id": 2470703,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4398, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662280.112, "dur": 342.052,
    "args": {
      "External id": 2470704,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662655.779, "dur": 2.105,
    "args": {
      "External id": 2470705,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4400, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662718.547, "dur": 6.956,
    "args": {
      "External id": 2470706,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662773.020, "dur": 1.516,
    "args": {
      "External id": 2470707,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662791.353, "dur": 1.021,
    "args": {
      "External id": 2470708,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662805.432, "dur": 3.134,
    "args": {
      "External id": 2470709,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662820.733, "dur": 0.968,
    "args": {
      "External id": 2470710,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662831.116, "dur": 0.992,
    "args": {
      "External id": 2470711,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662843.330, "dur": 0.807,
    "args": {
      "External id": 2470712,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662854.750, "dur": 2.616,
    "args": {
      "External id": 2470713,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662867.752, "dur": 0.630,
    "args": {
      "External id": 2470714,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662957.398, "dur": 2753.391,
    "args": {
      "External id": 2470715,"Record function id": 0, "Ev Idx": 4410
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.8)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657662976.497, "dur": 1001.624,
    "args": {
      "External id": 2470716,"Record function id": 0, "Ev Idx": 4411
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663028.068, "dur": 310.207,
    "args": {
      "External id": 2470717,"Record function id": 0, "Ev Idx": 4412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663109.322, "dur": 4.968,
    "args": {
      "External id": 2470718,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663117.084, "dur": 0.782,
    "args": {
      "External id": 2470719,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663119.642, "dur": 0.685,
    "args": {
      "External id": 2470720,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663122.250, "dur": 0.888,
    "args": {
      "External id": 2470721,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663124.502, "dur": 0.962,
    "args": {
      "External id": 2470722,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663127.170, "dur": 2.412,
    "args": {
      "External id": 2470723,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663131.022, "dur": 0.941,
    "args": {
      "External id": 2470724,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663133.962, "dur": 0.591,
    "args": {
      "External id": 2470725,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663137.665, "dur": 1.089,
    "args": {
      "External id": 2470726,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663139.991, "dur": 1.111,
    "args": {
      "External id": 2470727,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663159.874, "dur": 148.727,
    "args": {
      "External id": 2470728,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663176.271, "dur": 128.185,
    "args": {
      "External id": 2470729,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663194.855, "dur": 11.770,
    "args": {
      "External id": 2470730,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663209.687, "dur": 66.608,
    "args": {
      "External id": 2470731,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663213.985, "dur": 61.969,
    "args": {
      "External id": 2470732,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663218.168, "dur": 5.647,
    "args": {
      "External id": 2470733,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663225.537, "dur": 49.945,
    "args": {
      "External id": 2470734,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4429
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.7", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663413.758, "dur": 556.462,
    "args": {
      "External id": 2470735,"Record function id": 0, "Ev Idx": 4430
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663430.328, "dur": 527.655,
    "args": {
      "External id": 2470736,"Record function id": 0, "Ev Idx": 4431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663482.778, "dur": 4.242,
    "args": {
      "External id": 2470737,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663502.278, "dur": 50.320,
    "args": {
      "External id": 2470738,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663510.591, "dur": 3.204,
    "args": {
      "External id": 2470739,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663515.213, "dur": 0.583,
    "args": {
      "External id": 2470740,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663516.744, "dur": 0.795,
    "args": {
      "External id": 2470741,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663518.660, "dur": 0.520,
    "args": {
      "External id": 2470742,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663520.835, "dur": 0.601,
    "args": {
      "External id": 2470743,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663523.046, "dur": 0.544,
    "args": {
      "External id": 2470744,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663525.128, "dur": 0.335,
    "args": {
      "External id": 2470745,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663526.684, "dur": 0.609,
    "args": {
      "External id": 2470746,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663545.812, "dur": 2.885,
    "args": {
      "External id": 2470747,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663563.343, "dur": 35.723,
    "args": {
      "External id": 2470748,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663630.848, "dur": 101.871,
    "args": {
      "External id": 2470749,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663641.504, "dur": 3.900,
    "args": {
      "External id": 2470750,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663650.588, "dur": 10.489,
    "args": {
      "External id": 2470751,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663654.784, "dur": 5.866,
    "args": {
      "External id": 2470752,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663658.920, "dur": 0.599,
    "args": {
      "External id": 2470753,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663668.373, "dur": 24.654,
    "args": {
      "External id": 2470754,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663670.655, "dur": 0.612,
    "args": {
      "External id": 2470755,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663672.713, "dur": 0.343,
    "args": {
      "External id": 2470756,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663674.681, "dur": 0.594,
    "args": {
      "External id": 2470757,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663676.948, "dur": 0.695,
    "args": {
      "External id": 2470758,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663679.554, "dur": 0.542,
    "args": {
      "External id": 2470759,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663681.412, "dur": 2.391,
    "args": {
      "External id": 2470760,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663685.309, "dur": 0.314,
    "args": {
      "External id": 2470761,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663687.063, "dur": 0.476,
    "args": {
      "External id": 2470762,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663688.956, "dur": 0.553,
    "args": {
      "External id": 2470763,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663705.750, "dur": 19.210,
    "args": {
      "External id": 2470764,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663776.767, "dur": 114.314,
    "args": {
      "External id": 2470765,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663803.310, "dur": 84.445,
    "args": {
      "External id": 2470766,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4461, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663812.583, "dur": 71.203,
    "args": {
      "External id": 2470767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657663909.738, "dur": 1.846,
    "args": {
      "External id": 2470768,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4463, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664024.741, "dur": 1666.130,
    "args": {
      "External id": 2470769,"Sequence number": 24740438, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4464
    }
  },
  {
    "ph": "f", "id": 210, "pid": 1336756, "tid": 1381189, "ts": 1514657664024.741,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664136.013, "dur": 110.708,
    "args": {
      "External id": 2470770,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664291.712, "dur": 37.892,
    "args": {
      "External id": 2470771,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664345.149, "dur": 51.395,
    "args": {
      "External id": 2470772,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664406.769, "dur": 31.920,
    "args": {
      "External id": 2470773,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664444.576, "dur": 45.402,
    "args": {
      "External id": 2470774,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664498.552, "dur": 26.913,
    "args": {
      "External id": 2470775,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664552.443, "dur": 47.734,
    "args": {
      "External id": 2470776,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664626.868, "dur": 27.067,
    "args": {
      "External id": 2470777,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664671.756, "dur": 30.038,
    "args": {
      "External id": 2470778,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664723.751, "dur": 20.373,
    "args": {
      "External id": 2470779,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664760.367, "dur": 17.915,
    "args": {
      "External id": 2470780,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664786.211, "dur": 28.951,
    "args": {
      "External id": 2470781,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664818.403, "dur": 33.099,
    "args": {
      "External id": 2470782,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664880.789, "dur": 215.149,
    "args": {
      "External id": 2470783,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664956.128, "dur": 7.943,
    "args": {
      "External id": 2470784,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657664965.982, "dur": 2.052,
    "args": {
      "External id": 2470785,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665130.248, "dur": 26.520,
    "args": {
      "External id": 2470786,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665168.416, "dur": 15.578,
    "args": {
      "External id": 2470787,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665192.784, "dur": 55.407,
    "args": {
      "External id": 2470788,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665254.239, "dur": 35.551,
    "args": {
      "External id": 2470789,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665299.617, "dur": 22.247,
    "args": {
      "External id": 2470790,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665329.043, "dur": 30.074,
    "args": {
      "External id": 2470791,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665364.780, "dur": 21.898,
    "args": {
      "External id": 2470792,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665394.345, "dur": 30.938,
    "args": {
      "External id": 2470793,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665447.434, "dur": 36.407,
    "args": {
      "External id": 2470794,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665509.663, "dur": 41.159,
    "args": {
      "External id": 2470795,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665571.310, "dur": 18.985,
    "args": {
      "External id": 2470796,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665607.796, "dur": 19.831,
    "args": {
      "External id": 2470797,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665641.791, "dur": 17.493,
    "args": {
      "External id": 2470798,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665732.989, "dur": 19.756,
    "args": {
      "External id": 2470799,"Record function id": 0, "Ev Idx": 4494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665739.827, "dur": 11.800,
    "args": {
      "External id": 2470800,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665744.534, "dur": 6.189,
    "args": {
      "External id": 2470801,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665746.474, "dur": 4.132,
    "args": {
      "External id": 2470802,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665756.692, "dur": 5.494,
    "args": {
      "External id": 2470803,"Record function id": 0, "Ev Idx": 4498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665758.360, "dur": 3.394,
    "args": {
      "External id": 2470804,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665759.200, "dur": 2.083,
    "args": {
      "External id": 2470805,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665759.979, "dur": 1.235,
    "args": {
      "External id": 2470806,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665765.779, "dur": 4.407,
    "args": {
      "External id": 2470807,"Record function id": 0, "Ev Idx": 4502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665767.017, "dur": 2.726,
    "args": {
      "External id": 2470808,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665767.763, "dur": 1.563,
    "args": {
      "External id": 2470809,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665768.272, "dur": 0.982,
    "args": {
      "External id": 2470810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665773.567, "dur": 4.537,
    "args": {
      "External id": 2470811,"Record function id": 0, "Ev Idx": 4506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665774.753, "dur": 2.948,
    "args": {
      "External id": 2470812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665775.517, "dur": 1.803,
    "args": {
      "External id": 2470813,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665776.356, "dur": 0.896,
    "args": {
      "External id": 2470814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665781.447, "dur": 5.985,
    "args": {
      "External id": 2470815,"Record function id": 0, "Ev Idx": 4510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665782.816, "dur": 4.187,
    "args": {
      "External id": 2470816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665783.412, "dur": 3.210,
    "args": {
      "External id": 2470817,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665783.772, "dur": 2.790,
    "args": {
      "External id": 2470818,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665790.621, "dur": 3.898,
    "args": {
      "External id": 2470819,"Record function id": 0, "Ev Idx": 4514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665791.985, "dur": 2.100,
    "args": {
      "External id": 2470820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665792.436, "dur": 1.234,
    "args": {
      "External id": 2470821,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665792.885, "dur": 0.689,
    "args": {
      "External id": 2470822,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665801.146, "dur": 3.879,
    "args": {
      "External id": 2470823,"Record function id": 0, "Ev Idx": 4518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665802.376, "dur": 2.235,
    "args": {
      "External id": 2470824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665802.926, "dur": 1.271,
    "args": {
      "External id": 2470825,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665803.203, "dur": 0.897,
    "args": {
      "External id": 2470826,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665808.182, "dur": 4.014,
    "args": {
      "External id": 2470827,"Record function id": 0, "Ev Idx": 4522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665809.711, "dur": 2.063,
    "args": {
      "External id": 2470828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665810.195, "dur": 1.136,
    "args": {
      "External id": 2470829,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665810.454, "dur": 0.780,
    "args": {
      "External id": 2470830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665815.271, "dur": 3.682,
    "args": {
      "External id": 2470831,"Record function id": 0, "Ev Idx": 4526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665816.233, "dur": 2.285,
    "args": {
      "External id": 2470832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665816.804, "dur": 1.328,
    "args": {
      "External id": 2470833,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665817.445, "dur": 0.561,
    "args": {
      "External id": 2470834,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665822.540, "dur": 36140.590,
    "args": {
      "External id": 2470835,"Record function id": 0, "Sequence number": 24740437, "Fwd thread id": 1, "Ev Idx": 4530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665824.004, "dur": 36130.156,
    "args": {
      "External id": 2470836,"Sequence number": 24740437, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4531
    }
  },
  {
    "ph": "f", "id": 211, "pid": 1336756, "tid": 1381189, "ts": 1514657665824.004,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.8)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665851.931, "dur": 42.636,
    "args": {
      "External id": 2470837,"Record function id": 0, "Ev Idx": 4532
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.8)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665902.641, "dur": 67.942,
    "args": {
      "External id": 2470838,"Record function id": 0, "Ev Idx": 4533
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.8)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657665975.842, "dur": 35970.430,
    "args": {
      "External id": 2470839,"Record function id": 0, "Ev Idx": 4534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666112.741, "dur": 7.069,
    "args": {
      "External id": 2470840,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666130.483, "dur": 5.146,
    "args": {
      "External id": 2470841,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666150.616, "dur": 34941.515,
    "args": {
      "External id": 2470842,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666164.454, "dur": 34919.656,
    "args": {
      "External id": 2470843,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666212.070, "dur": 13.537,
    "args": {
      "External id": 2470844,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666232.336, "dur": 34814.166,
    "args": {
      "External id": 2470845,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666235.183, "dur": 34810.612,
    "args": {
      "External id": 2470846,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666239.392, "dur": 7.213,
    "args": {
      "External id": 2470847,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657666248.391, "dur": 34793.858,
    "args": {
      "External id": 2470848,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701177.185, "dur": 10.204,
    "args": {
      "External id": 2470849,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701179.733, "dur": 7.343,
    "args": {
      "External id": 2470850,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701216.329, "dur": 430.698,
    "args": {
      "External id": 2470851,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701244.600, "dur": 396.920,
    "args": {
      "External id": 2470852,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4547, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701255.442, "dur": 380.719,
    "args": {
      "External id": 2470853,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701671.402, "dur": 2.616,
    "args": {
      "External id": 2470854,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4549, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701736.305, "dur": 7.126,
    "args": {
      "External id": 2470855,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701789.000, "dur": 1.600,
    "args": {
      "External id": 2470856,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701806.232, "dur": 1.226,
    "args": {
      "External id": 2470857,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701818.735, "dur": 3.320,
    "args": {
      "External id": 2470858,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701837.547, "dur": 1.243,
    "args": {
      "External id": 2470859,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701849.257, "dur": 0.800,
    "args": {
      "External id": 2470860,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701860.678, "dur": 1.126,
    "args": {
      "External id": 2470861,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701872.462, "dur": 2.957,
    "args": {
      "External id": 2470862,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701885.302, "dur": 0.906,
    "args": {
      "External id": 2470863,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657701978.035, "dur": 2807.323,
    "args": {
      "External id": 2470864,"Record function id": 0, "Ev Idx": 4559
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.7)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702034.567, "dur": 1045.635,
    "args": {
      "External id": 2470865,"Record function id": 0, "Ev Idx": 4560
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702051.414, "dur": 311.694,
    "args": {
      "External id": 2470866,"Record function id": 0, "Ev Idx": 4561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702133.329, "dur": 4.571,
    "args": {
      "External id": 2470867,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702142.260, "dur": 1.196,
    "args": {
      "External id": 2470868,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702145.237, "dur": 1.235,
    "args": {
      "External id": 2470869,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702148.324, "dur": 1.032,
    "args": {
      "External id": 2470870,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702150.948, "dur": 0.865,
    "args": {
      "External id": 2470871,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702154.442, "dur": 2.253,
    "args": {
      "External id": 2470872,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702158.220, "dur": 0.796,
    "args": {
      "External id": 2470873,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702160.401, "dur": 1.033,
    "args": {
      "External id": 2470874,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702163.138, "dur": 0.786,
    "args": {
      "External id": 2470875,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702166.468, "dur": 0.981,
    "args": {
      "External id": 2470876,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702185.939, "dur": 147.307,
    "args": {
      "External id": 2470877,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702201.841, "dur": 127.052,
    "args": {
      "External id": 2470878,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702219.380, "dur": 12.536,
    "args": {
      "External id": 2470879,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702234.714, "dur": 65.662,
    "args": {
      "External id": 2470880,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702237.226, "dur": 62.826,
    "args": {
      "External id": 2470881,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702241.081, "dur": 6.177,
    "args": {
      "External id": 2470882,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702248.778, "dur": 50.845,
    "args": {
      "External id": 2470883,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4578
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.6", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702441.857, "dur": 631.154,
    "args": {
      "External id": 2470884,"Record function id": 0, "Ev Idx": 4579
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702457.208, "dur": 602.881,
    "args": {
      "External id": 2470885,"Record function id": 0, "Ev Idx": 4580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702510.077, "dur": 4.693,
    "args": {
      "External id": 2470886,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702551.287, "dur": 31.685,
    "args": {
      "External id": 2470887,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702556.087, "dur": 5.114,
    "args": {
      "External id": 2470888,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702563.545, "dur": 0.257,
    "args": {
      "External id": 2470889,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702565.127, "dur": 0.355,
    "args": {
      "External id": 2470890,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702568.489, "dur": 0.446,
    "args": {
      "External id": 2470891,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702570.329, "dur": 0.358,
    "args": {
      "External id": 2470892,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702571.961, "dur": 0.378,
    "args": {
      "External id": 2470893,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702573.688, "dur": 0.667,
    "args": {
      "External id": 2470894,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702575.908, "dur": 0.498,
    "args": {
      "External id": 2470895,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702577.554, "dur": 2.632,
    "args": {
      "External id": 2470896,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702593.882, "dur": 34.804,
    "args": {
      "External id": 2470897,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702661.155, "dur": 127.788,
    "args": {
      "External id": 2470898,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702671.664, "dur": 4.029,
    "args": {
      "External id": 2470899,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702681.164, "dur": 11.829,
    "args": {
      "External id": 2470900,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702687.111, "dur": 5.446,
    "args": {
      "External id": 2470901,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702690.760, "dur": 0.674,
    "args": {
      "External id": 2470902,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702701.055, "dur": 26.259,
    "args": {
      "External id": 2470903,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702703.348, "dur": 0.547,
    "args": {
      "External id": 2470904,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702705.761, "dur": 0.569,
    "args": {
      "External id": 2470905,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702707.854, "dur": 0.515,
    "args": {
      "External id": 2470906,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702710.292, "dur": 0.663,
    "args": {
      "External id": 2470907,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702712.365, "dur": 0.392,
    "args": {
      "External id": 2470908,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702714.441, "dur": 2.174,
    "args": {
      "External id": 2470909,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702717.914, "dur": 0.340,
    "args": {
      "External id": 2470910,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702719.712, "dur": 0.479,
    "args": {
      "External id": 2470911,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702721.702, "dur": 0.526,
    "args": {
      "External id": 2470912,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702758.591, "dur": 22.955,
    "args": {
      "External id": 2470913,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702834.139, "dur": 114.639,
    "args": {
      "External id": 2470914,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702859.886, "dur": 85.417,
    "args": {
      "External id": 2470915,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4610, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702869.548, "dur": 71.495,
    "args": {
      "External id": 2470916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657702964.462, "dur": 1.967,
    "args": {
      "External id": 2470917,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4612, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703088.124, "dur": 1672.431,
    "args": {
      "External id": 2470918,"Sequence number": 24740436, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4613
    }
  },
  {
    "ph": "f", "id": 212, "pid": 1336756, "tid": 1381189, "ts": 1514657703088.124,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703198.221, "dur": 108.982,
    "args": {
      "External id": 2470919,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703351.587, "dur": 40.415,
    "args": {
      "External id": 2470920,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703407.229, "dur": 49.764,
    "args": {
      "External id": 2470921,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703466.408, "dur": 32.368,
    "args": {
      "External id": 2470922,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703516.658, "dur": 65.327,
    "args": {
      "External id": 2470923,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703594.019, "dur": 30.141,
    "args": {
      "External id": 2470924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703632.287, "dur": 43.128,
    "args": {
      "External id": 2470925,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703701.760, "dur": 24.325,
    "args": {
      "External id": 2470926,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703745.539, "dur": 29.304,
    "args": {
      "External id": 2470927,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703793.658, "dur": 21.447,
    "args": {
      "External id": 2470928,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703831.448, "dur": 16.720,
    "args": {
      "External id": 2470929,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703857.843, "dur": 28.877,
    "args": {
      "External id": 2470930,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703889.833, "dur": 32.179,
    "args": {
      "External id": 2470931,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657703948.360, "dur": 219.924,
    "args": {
      "External id": 2470932,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704065.084, "dur": 12.299,
    "args": {
      "External id": 2470933,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704079.201, "dur": 2.678,
    "args": {
      "External id": 2470934,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704201.896, "dur": 24.581,
    "args": {
      "External id": 2470935,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704237.868, "dur": 15.929,
    "args": {
      "External id": 2470936,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704261.134, "dur": 40.495,
    "args": {
      "External id": 2470937,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704308.009, "dur": 47.068,
    "args": {
      "External id": 2470938,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704362.219, "dur": 23.810,
    "args": {
      "External id": 2470939,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704393.239, "dur": 46.294,
    "args": {
      "External id": 2470940,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704450.702, "dur": 24.893,
    "args": {
      "External id": 2470941,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704483.363, "dur": 32.545,
    "args": {
      "External id": 2470942,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704558.597, "dur": 26.750,
    "args": {
      "External id": 2470943,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704607.870, "dur": 23.284,
    "args": {
      "External id": 2470944,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704646.203, "dur": 21.352,
    "args": {
      "External id": 2470945,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704683.549, "dur": 15.517,
    "args": {
      "External id": 2470946,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704713.410, "dur": 20.832,
    "args": {
      "External id": 2470947,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704806.649, "dur": 14.756,
    "args": {
      "External id": 2470948,"Record function id": 0, "Ev Idx": 4643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704809.874, "dur": 10.699,
    "args": {
      "External id": 2470949,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704814.165, "dur": 5.508,
    "args": {
      "External id": 2470950,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704815.458, "dur": 4.106,
    "args": {
      "External id": 2470951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704825.353, "dur": 4.884,
    "args": {
      "External id": 2470952,"Record function id": 0, "Ev Idx": 4647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704826.897, "dur": 2.910,
    "args": {
      "External id": 2470953,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704827.759, "dur": 1.509,
    "args": {
      "External id": 2470954,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704828.472, "dur": 0.730,
    "args": {
      "External id": 2470955,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704833.564, "dur": 7.275,
    "args": {
      "External id": 2470956,"Record function id": 0, "Ev Idx": 4651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704834.950, "dur": 5.480,
    "args": {
      "External id": 2470957,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704835.670, "dur": 4.372,
    "args": {
      "External id": 2470958,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704839.210, "dur": 0.735,
    "args": {
      "External id": 2470959,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704844.027, "dur": 3.936,
    "args": {
      "External id": 2470960,"Record function id": 0, "Ev Idx": 4655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704845.400, "dur": 2.173,
    "args": {
      "External id": 2470961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704845.938, "dur": 1.208,
    "args": {
      "External id": 2470962,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704846.386, "dur": 0.691,
    "args": {
      "External id": 2470963,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704851.020, "dur": 5.344,
    "args": {
      "External id": 2470964,"Record function id": 0, "Ev Idx": 4659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704852.409, "dur": 3.560,
    "args": {
      "External id": 2470965,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704852.839, "dur": 2.726,
    "args": {
      "External id": 2470966,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704853.102, "dur": 2.400,
    "args": {
      "External id": 2470967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704859.400, "dur": 3.622,
    "args": {
      "External id": 2470968,"Record function id": 0, "Ev Idx": 4663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704860.366, "dur": 2.265,
    "args": {
      "External id": 2470969,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704860.883, "dur": 1.337,
    "args": {
      "External id": 2470970,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704861.250, "dur": 0.906,
    "args": {
      "External id": 2470971,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704866.162, "dur": 3.388,
    "args": {
      "External id": 2470972,"Record function id": 0, "Ev Idx": 4667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704867.125, "dur": 2.031,
    "args": {
      "External id": 2470973,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704867.622, "dur": 1.040,
    "args": {
      "External id": 2470974,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704867.975, "dur": 0.615,
    "args": {
      "External id": 2470975,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704872.597, "dur": 4.356,
    "args": {
      "External id": 2470976,"Record function id": 0, "Ev Idx": 4671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704873.708, "dur": 2.841,
    "args": {
      "External id": 2470977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704874.160, "dur": 1.937,
    "args": {
      "External id": 2470978,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704875.280, "dur": 0.741,
    "args": {
      "External id": 2470979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704879.977, "dur": 4.336,
    "args": {
      "External id": 2470980,"Record function id": 0, "Ev Idx": 4675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704881.435, "dur": 2.489,
    "args": {
      "External id": 2470981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704882.275, "dur": 1.229,
    "args": {
      "External id": 2470982,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704882.755, "dur": 0.673,
    "args": {
      "External id": 2470983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704887.756, "dur": 35030.306,
    "args": {
      "External id": 2470984,"Record function id": 0, "Sequence number": 24740435, "Fwd thread id": 1, "Ev Idx": 4679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704889.060, "dur": 35019.930,
    "args": {
      "External id": 2470985,"Sequence number": 24740435, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4680
    }
  },
  {
    "ph": "f", "id": 213, "pid": 1336756, "tid": 1381189, "ts": 1514657704889.060,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.7)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704920.908, "dur": 39.056,
    "args": {
      "External id": 2470986,"Record function id": 0, "Ev Idx": 4681
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.7)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657704967.519, "dur": 100.012,
    "args": {
      "External id": 2470987,"Record function id": 0, "Ev Idx": 4682
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.7)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705075.175, "dur": 34825.783,
    "args": {
      "External id": 2470988,"Record function id": 0, "Ev Idx": 4683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705165.269, "dur": 6.830,
    "args": {
      "External id": 2470989,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705181.755, "dur": 5.239,
    "args": {
      "External id": 2470990,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705200.933, "dur": 33903.903,
    "args": {
      "External id": 2470991,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705214.017, "dur": 33882.527,
    "args": {
      "External id": 2470992,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705261.183, "dur": 13.032,
    "args": {
      "External id": 2470993,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705283.320, "dur": 33773.294,
    "args": {
      "External id": 2470994,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705285.716, "dur": 33770.267,
    "args": {
      "External id": 2470995,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705289.739, "dur": 7.047,
    "args": {
      "External id": 2470996,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657705298.527, "dur": 33754.183,
    "args": {
      "External id": 2470997,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739190.074, "dur": 8.165,
    "args": {
      "External id": 2470998,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739192.593, "dur": 5.338,
    "args": {
      "External id": 2470999,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739228.730, "dur": 383.611,
    "args": {
      "External id": 2471000,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739258.330, "dur": 348.765,
    "args": {
      "External id": 2471001,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4696, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739268.795, "dur": 332.430,
    "args": {
      "External id": 2471002,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739635.048, "dur": 2.219,
    "args": {
      "External id": 2471003,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4698, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739698.097, "dur": 6.770,
    "args": {
      "External id": 2471004,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739750.738, "dur": 1.329,
    "args": {
      "External id": 2471005,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739767.131, "dur": 1.343,
    "args": {
      "External id": 2471006,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739779.867, "dur": 2.607,
    "args": {
      "External id": 2471007,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739794.461, "dur": 1.161,
    "args": {
      "External id": 2471008,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739806.235, "dur": 1.086,
    "args": {
      "External id": 2471009,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739818.108, "dur": 1.002,
    "args": {
      "External id": 2471010,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739830.229, "dur": 2.816,
    "args": {
      "External id": 2471011,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739843.219, "dur": 0.980,
    "args": {
      "External id": 2471012,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739931.714, "dur": 2712.960,
    "args": {
      "External id": 2471013,"Record function id": 0, "Ev Idx": 4708
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.6)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739950.892, "dur": 997.661,
    "args": {
      "External id": 2471014,"Record function id": 0, "Ev Idx": 4709
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657739963.721, "dur": 343.992,
    "args": {
      "External id": 2471015,"Record function id": 0, "Ev Idx": 4710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740077.062, "dur": 4.589,
    "args": {
      "External id": 2471016,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740084.732, "dur": 1.042,
    "args": {
      "External id": 2471017,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740087.533, "dur": 1.040,
    "args": {
      "External id": 2471018,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740091.325, "dur": 0.999,
    "args": {
      "External id": 2471019,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740093.641, "dur": 1.075,
    "args": {
      "External id": 2471020,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740096.148, "dur": 2.563,
    "args": {
      "External id": 2471021,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740100.504, "dur": 1.056,
    "args": {
      "External id": 2471022,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740103.584, "dur": 0.812,
    "args": {
      "External id": 2471023,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740105.789, "dur": 1.224,
    "args": {
      "External id": 2471024,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740108.851, "dur": 1.045,
    "args": {
      "External id": 2471025,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740129.391, "dur": 150.377,
    "args": {
      "External id": 2471026,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740145.754, "dur": 129.850,
    "args": {
      "External id": 2471027,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740165.963, "dur": 12.001,
    "args": {
      "External id": 2471028,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740180.986, "dur": 66.261,
    "args": {
      "External id": 2471029,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740183.283, "dur": 63.591,
    "args": {
      "External id": 2471030,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740189.157, "dur": 6.450,
    "args": {
      "External id": 2471031,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740197.456, "dur": 48.850,
    "args": {
      "External id": 2471032,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740382.650, "dur": 558.488,
    "args": {
      "External id": 2471033,"Record function id": 0, "Ev Idx": 4728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740397.887, "dur": 531.878,
    "args": {
      "External id": 2471034,"Record function id": 0, "Ev Idx": 4729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740449.217, "dur": 4.598,
    "args": {
      "External id": 2471035,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740472.839, "dur": 30.927,
    "args": {
      "External id": 2471036,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740477.442, "dur": 3.445,
    "args": {
      "External id": 2471037,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740483.608, "dur": 0.570,
    "args": {
      "External id": 2471038,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740485.567, "dur": 0.783,
    "args": {
      "External id": 2471039,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740488.055, "dur": 0.473,
    "args": {
      "External id": 2471040,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740489.950, "dur": 0.418,
    "args": {
      "External id": 2471041,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740492.619, "dur": 0.456,
    "args": {
      "External id": 2471042,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740494.541, "dur": 0.736,
    "args": {
      "External id": 2471043,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740496.748, "dur": 0.573,
    "args": {
      "External id": 2471044,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740498.778, "dur": 2.320,
    "args": {
      "External id": 2471045,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740514.041, "dur": 48.439,
    "args": {
      "External id": 2471046,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740596.260, "dur": 116.624,
    "args": {
      "External id": 2471047,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740606.903, "dur": 4.046,
    "args": {
      "External id": 2471048,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740615.935, "dur": 10.192,
    "args": {
      "External id": 2471049,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740619.901, "dur": 5.773,
    "args": {
      "External id": 2471050,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740623.628, "dur": 0.682,
    "args": {
      "External id": 2471051,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740633.207, "dur": 27.573,
    "args": {
      "External id": 2471052,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740635.804, "dur": 0.491,
    "args": {
      "External id": 2471053,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740637.730, "dur": 0.728,
    "args": {
      "External id": 2471054,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740642.667, "dur": 0.493,
    "args": {
      "External id": 2471055,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740644.530, "dur": 0.564,
    "args": {
      "External id": 2471056,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740646.501, "dur": 0.427,
    "args": {
      "External id": 2471057,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740648.422, "dur": 2.310,
    "args": {
      "External id": 2471058,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740652.298, "dur": 0.442,
    "args": {
      "External id": 2471059,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740654.175, "dur": 0.383,
    "args": {
      "External id": 2471060,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740655.807, "dur": 0.406,
    "args": {
      "External id": 2471061,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740679.146, "dur": 26.617,
    "args": {
      "External id": 2471062,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740756.881, "dur": 110.884,
    "args": {
      "External id": 2471063,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740782.744, "dur": 82.003,
    "args": {
      "External id": 2471064,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4759, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740792.052, "dur": 68.811,
    "args": {
      "External id": 2471065,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740881.798, "dur": 1.691,
    "args": {
      "External id": 2471066,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4761, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657740955.255, "dur": 1663.396,
    "args": {
      "External id": 2471067,"Sequence number": 24740434, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4762
    }
  },
  {
    "ph": "f", "id": 214, "pid": 1336756, "tid": 1381189, "ts": 1514657740955.255,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741120.093, "dur": 109.672,
    "args": {
      "External id": 2471068,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741272.990, "dur": 39.249,
    "args": {
      "External id": 2471069,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741327.457, "dur": 48.401,
    "args": {
      "External id": 2471070,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741385.855, "dur": 31.161,
    "args": {
      "External id": 2471071,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741423.018, "dur": 46.128,
    "args": {
      "External id": 2471072,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741477.633, "dur": 26.737,
    "args": {
      "External id": 2471073,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741511.771, "dur": 61.531,
    "args": {
      "External id": 2471074,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741598.629, "dur": 24.397,
    "args": {
      "External id": 2471075,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741641.142, "dur": 30.448,
    "args": {
      "External id": 2471076,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741689.446, "dur": 19.181,
    "args": {
      "External id": 2471077,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741724.410, "dur": 16.292,
    "args": {
      "External id": 2471078,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741748.454, "dur": 31.971,
    "args": {
      "External id": 2471079,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741783.392, "dur": 33.230,
    "args": {
      "External id": 2471080,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741847.997, "dur": 208.291,
    "args": {
      "External id": 2471081,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741923.647, "dur": 7.433,
    "args": {
      "External id": 2471082,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657741932.863, "dur": 2.867,
    "args": {
      "External id": 2471083,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742090.683, "dur": 26.004,
    "args": {
      "External id": 2471084,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742128.154, "dur": 15.700,
    "args": {
      "External id": 2471085,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742151.917, "dur": 41.221,
    "args": {
      "External id": 2471086,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742198.831, "dur": 36.192,
    "args": {
      "External id": 2471087,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742240.888, "dur": 21.720,
    "args": {
      "External id": 2471088,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742268.348, "dur": 29.171,
    "args": {
      "External id": 2471089,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742303.097, "dur": 21.234,
    "args": {
      "External id": 2471090,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742330.181, "dur": 31.065,
    "args": {
      "External id": 2471091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742379.004, "dur": 21.182,
    "args": {
      "External id": 2471092,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742419.409, "dur": 39.594,
    "args": {
      "External id": 2471093,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742480.380, "dur": 21.276,
    "args": {
      "External id": 2471094,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742518.582, "dur": 30.854,
    "args": {
      "External id": 2471095,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742570.065, "dur": 20.070,
    "args": {
      "External id": 2471096,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742666.522, "dur": 14.772,
    "args": {
      "External id": 2471097,"Record function id": 0, "Ev Idx": 4792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742669.949, "dur": 10.366,
    "args": {
      "External id": 2471098,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742674.229, "dur": 5.258,
    "args": {
      "External id": 2471099,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742675.551, "dur": 3.849,
    "args": {
      "External id": 2471100,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742688.449, "dur": 4.584,
    "args": {
      "External id": 2471101,"Record function id": 0, "Ev Idx": 4796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742689.832, "dur": 2.773,
    "args": {
      "External id": 2471102,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742690.624, "dur": 1.510,
    "args": {
      "External id": 2471103,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742691.317, "dur": 0.727,
    "args": {
      "External id": 2471104,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742696.505, "dur": 4.387,
    "args": {
      "External id": 2471105,"Record function id": 0, "Ev Idx": 4800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742697.826, "dur": 2.652,
    "args": {
      "External id": 2471106,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742698.394, "dur": 1.692,
    "args": {
      "External id": 2471107,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742699.215, "dur": 0.784,
    "args": {
      "External id": 2471108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742703.964, "dur": 3.435,
    "args": {
      "External id": 2471109,"Record function id": 0, "Ev Idx": 4804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742704.864, "dur": 2.127,
    "args": {
      "External id": 2471110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742705.564, "dur": 1.044,
    "args": {
      "External id": 2471111,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742705.919, "dur": 0.610,
    "args": {
      "External id": 2471112,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742710.461, "dur": 5.737,
    "args": {
      "External id": 2471113,"Record function id": 0, "Ev Idx": 4808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742711.535, "dur": 4.248,
    "args": {
      "External id": 2471114,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742712.015, "dur": 3.373,
    "args": {
      "External id": 2471115,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742712.479, "dur": 2.817,
    "args": {
      "External id": 2471116,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742719.182, "dur": 4.675,
    "args": {
      "External id": 2471117,"Record function id": 0, "Ev Idx": 4812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742720.142, "dur": 3.281,
    "args": {
      "External id": 2471118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742720.607, "dur": 2.406,
    "args": {
      "External id": 2471119,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742722.171, "dur": 0.744,
    "args": {
      "External id": 2471120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742727.028, "dur": 3.387,
    "args": {
      "External id": 2471121,"Record function id": 0, "Ev Idx": 4816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742728.168, "dur": 1.834,
    "args": {
      "External id": 2471122,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742728.659, "dur": 0.952,
    "args": {
      "External id": 2471123,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742728.921, "dur": 0.593,
    "args": {
      "External id": 2471124,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742733.471, "dur": 3.488,
    "args": {
      "External id": 2471125,"Record function id": 0, "Ev Idx": 4820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742734.529, "dur": 1.993,
    "args": {
      "External id": 2471126,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742734.986, "dur": 1.106,
    "args": {
      "External id": 2471127,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742735.401, "dur": 0.625,
    "args": {
      "External id": 2471128,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742739.965, "dur": 7.390,
    "args": {
      "External id": 2471129,"Record function id": 0, "Ev Idx": 4824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742744.592, "dur": 2.356,
    "args": {
      "External id": 2471130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742745.240, "dur": 1.324,
    "args": {
      "External id": 2471131,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742745.817, "dur": 0.668,
    "args": {
      "External id": 2471132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742750.870, "dur": 36513.109,
    "args": {
      "External id": 2471133,"Record function id": 0, "Sequence number": 24740433, "Fwd thread id": 1, "Ev Idx": 4828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742752.387, "dur": 36503.257,
    "args": {
      "External id": 2471134,"Sequence number": 24740433, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4829
    }
  },
  {
    "ph": "f", "id": 215, "pid": 1336756, "tid": 1381189, "ts": 1514657742752.387,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.6)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742780.868, "dur": 39.682,
    "args": {
      "External id": 2471135,"Record function id": 0, "Ev Idx": 4830
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.6)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742828.033, "dur": 64.073,
    "args": {
      "External id": 2471136,"Record function id": 0, "Ev Idx": 4831
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.6)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657742898.089, "dur": 36349.525,
    "args": {
      "External id": 2471137,"Record function id": 0, "Ev Idx": 4832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743019.331, "dur": 8.182,
    "args": {
      "External id": 2471138,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743042.671, "dur": 5.005,
    "args": {
      "External id": 2471139,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743063.247, "dur": 35347.660,
    "args": {
      "External id": 2471140,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743076.999, "dur": 35325.866,
    "args": {
      "External id": 2471141,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743120.259, "dur": 14.012,
    "args": {
      "External id": 2471142,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743140.373, "dur": 35224.465,
    "args": {
      "External id": 2471143,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743142.625, "dur": 35221.527,
    "args": {
      "External id": 2471144,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743147.046, "dur": 8.421,
    "args": {
      "External id": 2471145,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657743158.805, "dur": 35202.016,
    "args": {
      "External id": 2471146,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657778494.514, "dur": 7.835,
    "args": {
      "External id": 2471147,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657778497.092, "dur": 4.967,
    "args": {
      "External id": 2471148,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657778544.353, "dur": 369.079,
    "args": {
      "External id": 2471149,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657778572.930, "dur": 335.445,
    "args": {
      "External id": 2471150,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4845, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657778583.840, "dur": 318.640,
    "args": {
      "External id": 2471151,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657778935.496, "dur": 1.798,
    "args": {
      "External id": 2471152,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4847, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779042.820, "dur": 8.399,
    "args": {
      "External id": 2471153,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779096.022, "dur": 1.429,
    "args": {
      "External id": 2471154,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779111.358, "dur": 1.364,
    "args": {
      "External id": 2471155,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779123.962, "dur": 0.816,
    "args": {
      "External id": 2471156,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779136.529, "dur": 2.663,
    "args": {
      "External id": 2471157,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779148.932, "dur": 1.102,
    "args": {
      "External id": 2471158,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779161.151, "dur": 0.971,
    "args": {
      "External id": 2471159,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779172.307, "dur": 1.020,
    "args": {
      "External id": 2471160,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779182.706, "dur": 2.415,
    "args": {
      "External id": 2471161,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779279.593, "dur": 2755.129,
    "args": {
      "External id": 2471162,"Record function id": 0, "Ev Idx": 4857
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.5)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779299.713, "dur": 1007.782,
    "args": {
      "External id": 2471163,"Record function id": 0, "Ev Idx": 4858
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779315.128, "dur": 327.568,
    "args": {
      "External id": 2471164,"Record function id": 0, "Ev Idx": 4859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779393.150, "dur": 3.961,
    "args": {
      "External id": 2471165,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779400.230, "dur": 0.899,
    "args": {
      "External id": 2471166,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779404.180, "dur": 0.829,
    "args": {
      "External id": 2471167,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779406.804, "dur": 0.881,
    "args": {
      "External id": 2471168,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779409.253, "dur": 0.787,
    "args": {
      "External id": 2471169,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779411.425, "dur": 1.073,
    "args": {
      "External id": 2471170,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779415.598, "dur": 0.771,
    "args": {
      "External id": 2471171,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779417.806, "dur": 2.496,
    "args": {
      "External id": 2471172,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779421.753, "dur": 1.107,
    "args": {
      "External id": 2471173,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779424.263, "dur": 0.783,
    "args": {
      "External id": 2471174,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779445.215, "dur": 168.035,
    "args": {
      "External id": 2471175,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779461.409, "dur": 147.507,
    "args": {
      "External id": 2471176,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779480.600, "dur": 13.534,
    "args": {
      "External id": 2471177,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779497.221, "dur": 81.710,
    "args": {
      "External id": 2471178,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779499.855, "dur": 78.646,
    "args": {
      "External id": 2471179,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779503.862, "dur": 6.685,
    "args": {
      "External id": 2471180,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779512.213, "dur": 65.379,
    "args": {
      "External id": 2471181,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4876
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779722.528, "dur": 577.225,
    "args": {
      "External id": 2471182,"Record function id": 0, "Ev Idx": 4877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779738.569, "dur": 549.689,
    "args": {
      "External id": 2471183,"Record function id": 0, "Ev Idx": 4878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779794.869, "dur": 5.063,
    "args": {
      "External id": 2471184,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779818.836, "dur": 29.428,
    "args": {
      "External id": 2471185,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779823.520, "dur": 2.349,
    "args": {
      "External id": 2471186,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779827.878, "dur": 0.460,
    "args": {
      "External id": 2471187,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779829.680, "dur": 2.046,
    "args": {
      "External id": 2471188,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779834.481, "dur": 0.508,
    "args": {
      "External id": 2471189,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779836.275, "dur": 0.503,
    "args": {
      "External id": 2471190,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779838.208, "dur": 0.305,
    "args": {
      "External id": 2471191,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779840.320, "dur": 0.507,
    "args": {
      "External id": 2471192,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779842.898, "dur": 0.319,
    "args": {
      "External id": 2471193,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779844.580, "dur": 1.118,
    "args": {
      "External id": 2471194,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779859.006, "dur": 31.271,
    "args": {
      "External id": 2471195,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779919.841, "dur": 143.879,
    "args": {
      "External id": 2471196,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779929.915, "dur": 3.173,
    "args": {
      "External id": 2471197,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779938.168, "dur": 11.107,
    "args": {
      "External id": 2471198,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779942.149, "dur": 6.704,
    "args": {
      "External id": 2471199,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779945.697, "dur": 1.978,
    "args": {
      "External id": 2471200,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779956.191, "dur": 60.145,
    "args": {
      "External id": 2471201,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779958.260, "dur": 0.644,
    "args": {
      "External id": 2471202,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779960.852, "dur": 0.507,
    "args": {
      "External id": 2471203,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779963.005, "dur": 1.211,
    "args": {
      "External id": 2471204,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779965.664, "dur": 0.455,
    "args": {
      "External id": 2471205,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779967.652, "dur": 0.412,
    "args": {
      "External id": 2471206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779969.772, "dur": 0.236,
    "args": {
      "External id": 2471207,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779971.186, "dur": 0.329,
    "args": {
      "External id": 2471208,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779972.897, "dur": 2.241,
    "args": {
      "External id": 2471209,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657779976.418, "dur": 0.321,
    "args": {
      "External id": 2471210,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780031.803, "dur": 24.239,
    "args": {
      "External id": 2471211,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780108.991, "dur": 114.607,
    "args": {
      "External id": 2471212,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780135.458, "dur": 84.867,
    "args": {
      "External id": 2471213,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4908, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780144.579, "dur": 71.153,
    "args": {
      "External id": 2471214,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780239.206, "dur": 1.862,
    "args": {
      "External id": 2471215,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4910, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780314.080, "dur": 1661.997,
    "args": {
      "External id": 2471216,"Sequence number": 24740432, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4911
    }
  },
  {
    "ph": "f", "id": 216, "pid": 1336756, "tid": 1381189, "ts": 1514657780314.080,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780419.781, "dur": 102.732,
    "args": {
      "External id": 2471217,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780586.794, "dur": 41.648,
    "args": {
      "External id": 2471218,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780647.073, "dur": 55.406,
    "args": {
      "External id": 2471219,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780711.799, "dur": 32.674,
    "args": {
      "External id": 2471220,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780750.850, "dur": 45.156,
    "args": {
      "External id": 2471221,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780802.451, "dur": 27.806,
    "args": {
      "External id": 2471222,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780842.942, "dur": 41.649,
    "args": {
      "External id": 2471223,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780907.425, "dur": 23.674,
    "args": {
      "External id": 2471224,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657780949.283, "dur": 30.933,
    "args": {
      "External id": 2471225,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781040.646, "dur": 23.257,
    "args": {
      "External id": 2471226,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781079.706, "dur": 16.956,
    "args": {
      "External id": 2471227,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781107.040, "dur": 35.050,
    "args": {
      "External id": 2471228,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781148.218, "dur": 33.831,
    "args": {
      "External id": 2471229,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781211.956, "dur": 171.973,
    "args": {
      "External id": 2471230,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781291.794, "dur": 6.130,
    "args": {
      "External id": 2471231,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781299.960, "dur": 2.898,
    "args": {
      "External id": 2471232,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781416.247, "dur": 26.187,
    "args": {
      "External id": 2471233,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781460.400, "dur": 15.291,
    "args": {
      "External id": 2471234,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781483.751, "dur": 35.768,
    "args": {
      "External id": 2471235,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781526.549, "dur": 62.489,
    "args": {
      "External id": 2471236,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781599.262, "dur": 25.088,
    "args": {
      "External id": 2471237,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781628.643, "dur": 29.955,
    "args": {
      "External id": 2471238,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781664.335, "dur": 20.830,
    "args": {
      "External id": 2471239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781701.506, "dur": 43.505,
    "args": {
      "External id": 2471240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781775.121, "dur": 27.114,
    "args": {
      "External id": 2471241,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781824.585, "dur": 26.043,
    "args": {
      "External id": 2471242,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781864.249, "dur": 20.788,
    "args": {
      "External id": 2471243,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781900.123, "dur": 16.100,
    "args": {
      "External id": 2471244,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657781927.567, "dur": 20.830,
    "args": {
      "External id": 2471245,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782060.637, "dur": 14.950,
    "args": {
      "External id": 2471246,"Record function id": 0, "Ev Idx": 4941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782064.380, "dur": 10.248,
    "args": {
      "External id": 2471247,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782068.175, "dur": 5.591,
    "args": {
      "External id": 2471248,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782069.606, "dur": 4.035,
    "args": {
      "External id": 2471249,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782079.310, "dur": 6.642,
    "args": {
      "External id": 2471250,"Record function id": 0, "Ev Idx": 4945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782080.715, "dur": 4.773,
    "args": {
      "External id": 2471251,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782081.350, "dur": 3.647,
    "args": {
      "External id": 2471252,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782082.121, "dur": 2.746,
    "args": {
      "External id": 2471253,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782089.174, "dur": 4.215,
    "args": {
      "External id": 2471254,"Record function id": 0, "Ev Idx": 4949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782090.424, "dur": 2.563,
    "args": {
      "External id": 2471255,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782090.865, "dur": 1.656,
    "args": {
      "External id": 2471256,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782091.461, "dur": 0.979,
    "args": {
      "External id": 2471257,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782096.536, "dur": 3.390,
    "args": {
      "External id": 2471258,"Record function id": 0, "Ev Idx": 4953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782097.568, "dur": 1.959,
    "args": {
      "External id": 2471259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782098.075, "dur": 1.017,
    "args": {
      "External id": 2471260,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782098.488, "dur": 0.530,
    "args": {
      "External id": 2471261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782102.973, "dur": 6.510,
    "args": {
      "External id": 2471262,"Record function id": 0, "Ev Idx": 4957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782107.171, "dur": 1.916,
    "args": {
      "External id": 2471263,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782107.645, "dur": 0.942,
    "args": {
      "External id": 2471264,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782107.901, "dur": 0.624,
    "args": {
      "External id": 2471265,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782112.540, "dur": 3.667,
    "args": {
      "External id": 2471266,"Record function id": 0, "Ev Idx": 4961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782113.609, "dur": 2.164,
    "args": {
      "External id": 2471267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782114.107, "dur": 1.199,
    "args": {
      "External id": 2471268,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782114.513, "dur": 0.717,
    "args": {
      "External id": 2471269,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782119.413, "dur": 4.688,
    "args": {
      "External id": 2471270,"Record function id": 0, "Ev Idx": 4965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782120.480, "dur": 3.224,
    "args": {
      "External id": 2471271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782120.917, "dur": 2.351,
    "args": {
      "External id": 2471272,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782122.439, "dur": 0.752,
    "args": {
      "External id": 2471273,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782127.146, "dur": 3.276,
    "args": {
      "External id": 2471274,"Record function id": 0, "Ev Idx": 4969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782128.054, "dur": 1.965,
    "args": {
      "External id": 2471275,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782128.518, "dur": 1.036,
    "args": {
      "External id": 2471276,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782128.830, "dur": 0.650,
    "args": {
      "External id": 2471277,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782133.372, "dur": 4.862,
    "args": {
      "External id": 2471278,"Record function id": 0, "Ev Idx": 4973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782134.300, "dur": 3.537,
    "args": {
      "External id": 2471279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782134.744, "dur": 2.683,
    "args": {
      "External id": 2471280,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782135.025, "dur": 2.340,
    "args": {
      "External id": 2471281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782141.809, "dur": 37897.319,
    "args": {
      "External id": 2471282,"Record function id": 0, "Sequence number": 24740431, "Fwd thread id": 1, "Ev Idx": 4977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782143.551, "dur": 37885.395,
    "args": {
      "External id": 2471283,"Sequence number": 24740431, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4978
    }
  },
  {
    "ph": "f", "id": 217, "pid": 1336756, "tid": 1381189, "ts": 1514657782143.551,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.5)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782176.167, "dur": 38.187,
    "args": {
      "External id": 2471284,"Record function id": 0, "Ev Idx": 4979
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.5)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782221.954, "dur": 63.197,
    "args": {
      "External id": 2471285,"Record function id": 0, "Ev Idx": 4980
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.5)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782291.793, "dur": 37727.706,
    "args": {
      "External id": 2471286,"Record function id": 0, "Ev Idx": 4981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782378.140, "dur": 6.874,
    "args": {
      "External id": 2471287,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782394.875, "dur": 4.539,
    "args": {
      "External id": 2471288,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782417.370, "dur": 36747.643,
    "args": {
      "External id": 2471289,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782430.561, "dur": 36725.783,
    "args": {
      "External id": 2471290,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782471.582, "dur": 13.079,
    "args": {
      "External id": 2471291,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782490.522, "dur": 36628.860,
    "args": {
      "External id": 2471292,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782493.252, "dur": 36625.492,
    "args": {
      "External id": 2471293,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782497.332, "dur": 5.146,
    "args": {
      "External id": 2471294,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657782504.020, "dur": 36611.333,
    "args": {
      "External id": 2471295,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819249.099, "dur": 7.970,
    "args": {
      "External id": 2471296,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819251.683, "dur": 5.042,
    "args": {
      "External id": 2471297,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819286.989, "dur": 407.634,
    "args": {
      "External id": 2471298,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819317.298, "dur": 371.991,
    "args": {
      "External id": 2471299,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4994, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819328.020, "dur": 355.393,
    "args": {
      "External id": 2471300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819718.876, "dur": 2.578,
    "args": {
      "External id": 2471301,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4996, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819782.292, "dur": 6.701,
    "args": {
      "External id": 2471302,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819834.582, "dur": 1.277,
    "args": {
      "External id": 2471303,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819853.357, "dur": 1.418,
    "args": {
      "External id": 2471304,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819866.660, "dur": 0.814,
    "args": {
      "External id": 2471305,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819878.603, "dur": 0.781,
    "args": {
      "External id": 2471306,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819889.285, "dur": 0.742,
    "args": {
      "External id": 2471307,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819903.270, "dur": 1.293,
    "args": {
      "External id": 2471308,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819915.353, "dur": 1.199,
    "args": {
      "External id": 2471309,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657819925.752, "dur": 0.843,
    "args": {
      "External id": 2471310,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820054.970, "dur": 2685.518,
    "args": {
      "External id": 2471311,"Record function id": 0, "Ev Idx": 5006
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.4)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820075.275, "dur": 1008.765,
    "args": {
      "External id": 2471312,"Record function id": 0, "Ev Idx": 5007
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820087.971, "dur": 309.128,
    "args": {
      "External id": 2471313,"Record function id": 0, "Ev Idx": 5008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820166.415, "dur": 4.700,
    "args": {
      "External id": 2471314,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820174.139, "dur": 1.027,
    "args": {
      "External id": 2471315,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820178.792, "dur": 2.591,
    "args": {
      "External id": 2471316,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820183.114, "dur": 0.872,
    "args": {
      "External id": 2471317,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820185.562, "dur": 1.086,
    "args": {
      "External id": 2471318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820188.465, "dur": 0.717,
    "args": {
      "External id": 2471319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820192.206, "dur": 0.760,
    "args": {
      "External id": 2471320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820194.306, "dur": 0.805,
    "args": {
      "External id": 2471321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820196.558, "dur": 1.017,
    "args": {
      "External id": 2471322,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820199.637, "dur": 1.539,
    "args": {
      "External id": 2471323,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820221.368, "dur": 147.115,
    "args": {
      "External id": 2471324,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820237.430, "dur": 126.855,
    "args": {
      "External id": 2471325,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820255.002, "dur": 14.482,
    "args": {
      "External id": 2471326,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820272.741, "dur": 63.508,
    "args": {
      "External id": 2471327,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820275.228, "dur": 60.726,
    "args": {
      "External id": 2471328,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820279.499, "dur": 5.422,
    "args": {
      "External id": 2471329,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820286.722, "dur": 48.845,
    "args": {
      "External id": 2471330,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5025
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820473.018, "dur": 603.551,
    "args": {
      "External id": 2471331,"Record function id": 0, "Ev Idx": 5026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820489.817, "dur": 573.187,
    "args": {
      "External id": 2471332,"Record function id": 0, "Ev Idx": 5027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820564.081, "dur": 6.336,
    "args": {
      "External id": 2471333,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820589.410, "dur": 29.303,
    "args": {
      "External id": 2471334,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820594.365, "dur": 1.664,
    "args": {
      "External id": 2471335,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820599.212, "dur": 0.408,
    "args": {
      "External id": 2471336,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820601.241, "dur": 0.345,
    "args": {
      "External id": 2471337,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820603.202, "dur": 1.136,
    "args": {
      "External id": 2471338,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820605.452, "dur": 0.490,
    "args": {
      "External id": 2471339,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820607.208, "dur": 2.569,
    "args": {
      "External id": 2471340,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820612.358, "dur": 0.326,
    "args": {
      "External id": 2471341,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820614.051, "dur": 0.229,
    "args": {
      "External id": 2471342,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820615.512, "dur": 0.354,
    "args": {
      "External id": 2471343,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820630.688, "dur": 33.641,
    "args": {
      "External id": 2471344,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820694.914, "dur": 101.501,
    "args": {
      "External id": 2471345,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820704.975, "dur": 3.251,
    "args": {
      "External id": 2471346,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820713.142, "dur": 11.343,
    "args": {
      "External id": 2471347,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820717.319, "dur": 6.705,
    "args": {
      "External id": 2471348,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820721.326, "dur": 1.513,
    "args": {
      "External id": 2471349,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820730.812, "dur": 26.616,
    "args": {
      "External id": 2471350,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820732.927, "dur": 0.470,
    "args": {
      "External id": 2471351,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820735.303, "dur": 0.451,
    "args": {
      "External id": 2471352,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820737.393, "dur": 2.007,
    "args": {
      "External id": 2471353,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820741.176, "dur": 0.249,
    "args": {
      "External id": 2471354,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820743.046, "dur": 0.525,
    "args": {
      "External id": 2471355,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820744.945, "dur": 0.894,
    "args": {
      "External id": 2471356,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820747.717, "dur": 0.492,
    "args": {
      "External id": 2471357,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820749.771, "dur": 0.456,
    "args": {
      "External id": 2471358,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820751.747, "dur": 0.278,
    "args": {
      "External id": 2471359,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820769.478, "dur": 19.739,
    "args": {
      "External id": 2471360,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820840.160, "dur": 115.497,
    "args": {
      "External id": 2471361,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820867.851, "dur": 84.607,
    "args": {
      "External id": 2471362,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5057, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820876.753, "dur": 71.360,
    "args": {
      "External id": 2471363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657820970.887, "dur": 1.756,
    "args": {
      "External id": 2471364,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5059, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821091.169, "dur": 1629.605,
    "args": {
      "External id": 2471365,"Sequence number": 24740430, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5060
    }
  },
  {
    "ph": "f", "id": 218, "pid": 1336756, "tid": 1381189, "ts": 1514657821091.169,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821207.104, "dur": 107.148,
    "args": {
      "External id": 2471366,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821355.154, "dur": 38.329,
    "args": {
      "External id": 2471367,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821408.739, "dur": 50.118,
    "args": {
      "External id": 2471368,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821468.264, "dur": 31.857,
    "args": {
      "External id": 2471369,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821508.358, "dur": 62.991,
    "args": {
      "External id": 2471370,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821581.164, "dur": 30.153,
    "args": {
      "External id": 2471371,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821618.186, "dur": 42.369,
    "args": {
      "External id": 2471372,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821684.732, "dur": 25.785,
    "args": {
      "External id": 2471373,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821728.981, "dur": 30.134,
    "args": {
      "External id": 2471374,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821780.322, "dur": 19.625,
    "args": {
      "External id": 2471375,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821814.365, "dur": 17.136,
    "args": {
      "External id": 2471376,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821839.258, "dur": 29.474,
    "args": {
      "External id": 2471377,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821871.664, "dur": 33.433,
    "args": {
      "External id": 2471378,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657821933.657, "dur": 218.728,
    "args": {
      "External id": 2471379,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822054.370, "dur": 6.662,
    "args": {
      "External id": 2471380,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822063.110, "dur": 2.641,
    "args": {
      "External id": 2471381,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822189.252, "dur": 25.893,
    "args": {
      "External id": 2471382,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822226.249, "dur": 16.422,
    "args": {
      "External id": 2471383,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822250.344, "dur": 40.933,
    "args": {
      "External id": 2471384,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822296.844, "dur": 35.693,
    "args": {
      "External id": 2471385,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822338.835, "dur": 20.897,
    "args": {
      "External id": 2471386,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822365.314, "dur": 29.729,
    "args": {
      "External id": 2471387,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822400.312, "dur": 24.042,
    "args": {
      "External id": 2471388,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822430.676, "dur": 29.792,
    "args": {
      "External id": 2471389,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822494.445, "dur": 26.394,
    "args": {
      "External id": 2471390,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822555.377, "dur": 25.207,
    "args": {
      "External id": 2471391,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822602.304, "dur": 20.781,
    "args": {
      "External id": 2471392,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822640.825, "dur": 16.010,
    "args": {
      "External id": 2471393,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822675.211, "dur": 17.168,
    "args": {
      "External id": 2471394,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822761.832, "dur": 15.224,
    "args": {
      "External id": 2471395,"Record function id": 0, "Ev Idx": 5090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822764.888, "dur": 11.199,
    "args": {
      "External id": 2471396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822769.330, "dur": 5.848,
    "args": {
      "External id": 2471397,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822770.810, "dur": 4.246,
    "args": {
      "External id": 2471398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822780.812, "dur": 5.217,
    "args": {
      "External id": 2471399,"Record function id": 0, "Ev Idx": 5094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822782.658, "dur": 2.929,
    "args": {
      "External id": 2471400,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822783.518, "dur": 1.596,
    "args": {
      "External id": 2471401,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822783.965, "dur": 1.046,
    "args": {
      "External id": 2471402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822789.263, "dur": 4.970,
    "args": {
      "External id": 2471403,"Record function id": 0, "Ev Idx": 5098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822790.624, "dur": 3.178,
    "args": {
      "External id": 2471404,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822791.510, "dur": 1.867,
    "args": {
      "External id": 2471405,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822792.096, "dur": 1.195,
    "args": {
      "External id": 2471406,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822797.330, "dur": 3.860,
    "args": {
      "External id": 2471407,"Record function id": 0, "Ev Idx": 5102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822798.516, "dur": 2.265,
    "args": {
      "External id": 2471408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822798.998, "dur": 1.352,
    "args": {
      "External id": 2471409,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822799.608, "dur": 0.673,
    "args": {
      "External id": 2471410,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822804.244, "dur": 5.457,
    "args": {
      "External id": 2471411,"Record function id": 0, "Ev Idx": 5106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822805.584, "dur": 3.724,
    "args": {
      "External id": 2471412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822806.033, "dur": 2.838,
    "args": {
      "External id": 2471413,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822806.603, "dur": 2.199,
    "args": {
      "External id": 2471414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822812.774, "dur": 7.491,
    "args": {
      "External id": 2471415,"Record function id": 0, "Ev Idx": 5110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822816.940, "dur": 2.930,
    "args": {
      "External id": 2471416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822817.363, "dur": 2.077,
    "args": {
      "External id": 2471417,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822818.678, "dur": 0.696,
    "args": {
      "External id": 2471418,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822823.614, "dur": 3.628,
    "args": {
      "External id": 2471419,"Record function id": 0, "Ev Idx": 5114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822824.638, "dur": 2.213,
    "args": {
      "External id": 2471420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822825.390, "dur": 1.039,
    "args": {
      "External id": 2471421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822825.774, "dur": 0.579,
    "args": {
      "External id": 2471422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822830.269, "dur": 25.258,
    "args": {
      "External id": 2471423,"Record function id": 0, "Ev Idx": 5118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822831.291, "dur": 23.814,
    "args": {
      "External id": 2471424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822853.449, "dur": 1.082,
    "args": {
      "External id": 2471425,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822853.817, "dur": 0.641,
    "args": {
      "External id": 2471426,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822858.649, "dur": 4.344,
    "args": {
      "External id": 2471427,"Record function id": 0, "Ev Idx": 5122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822859.842, "dur": 2.734,
    "args": {
      "External id": 2471428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822860.290, "dur": 1.835,
    "args": {
      "External id": 2471429,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822861.416, "dur": 0.605,
    "args": {
      "External id": 2471430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822866.896, "dur": 35239.311,
    "args": {
      "External id": 2471431,"Record function id": 0, "Sequence number": 24740429, "Fwd thread id": 1, "Ev Idx": 5126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822868.571, "dur": 35228.381,
    "args": {
      "External id": 2471432,"Sequence number": 24740429, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5127
    }
  },
  {
    "ph": "f", "id": 219, "pid": 1336756, "tid": 1381189, "ts": 1514657822868.571,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.4)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822898.536, "dur": 36.424,
    "args": {
      "External id": 2471433,"Record function id": 0, "Ev Idx": 5128
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.4)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657822941.916, "dur": 99.809,
    "args": {
      "External id": 2471434,"Record function id": 0, "Ev Idx": 5129
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.4)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823049.392, "dur": 35039.395,
    "args": {
      "External id": 2471435,"Record function id": 0, "Ev Idx": 5130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823137.203, "dur": 7.007,
    "args": {
      "External id": 2471436,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823155.107, "dur": 5.261,
    "args": {
      "External id": 2471437,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823175.222, "dur": 34107.703,
    "args": {
      "External id": 2471438,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823188.145, "dur": 34085.944,
    "args": {
      "External id": 2471439,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823225.707, "dur": 13.093,
    "args": {
      "External id": 2471440,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823246.551, "dur": 33985.183,
    "args": {
      "External id": 2471441,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823249.068, "dur": 33981.922,
    "args": {
      "External id": 2471442,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823253.071, "dur": 7.270,
    "args": {
      "External id": 2471443,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657823262.021, "dur": 33965.273,
    "args": {
      "External id": 2471444,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857366.472, "dur": 8.583,
    "args": {
      "External id": 2471445,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857369.155, "dur": 5.577,
    "args": {
      "External id": 2471446,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857405.205, "dur": 361.212,
    "args": {
      "External id": 2471447,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857434.764, "dur": 326.652,
    "args": {
      "External id": 2471448,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5143, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857445.628, "dur": 310.259,
    "args": {
      "External id": 2471449,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857788.561, "dur": 2.312,
    "args": {
      "External id": 2471450,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5145, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857847.921, "dur": 6.502,
    "args": {
      "External id": 2471451,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857897.222, "dur": 1.567,
    "args": {
      "External id": 2471452,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857914.153, "dur": 1.349,
    "args": {
      "External id": 2471453,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857926.854, "dur": 2.781,
    "args": {
      "External id": 2471454,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857941.485, "dur": 0.682,
    "args": {
      "External id": 2471455,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857952.022, "dur": 0.874,
    "args": {
      "External id": 2471456,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857964.573, "dur": 0.876,
    "args": {
      "External id": 2471457,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657857975.949, "dur": 2.732,
    "args": {
      "External id": 2471458,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858026.856, "dur": 1.706,
    "args": {
      "External id": 2471459,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858121.794, "dur": 2663.441,
    "args": {
      "External id": 2471460,"Record function id": 0, "Ev Idx": 5155
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.3)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858139.956, "dur": 995.694,
    "args": {
      "External id": 2471461,"Record function id": 0, "Ev Idx": 5156
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858153.241, "dur": 299.479,
    "args": {
      "External id": 2471462,"Record function id": 0, "Ev Idx": 5157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858229.023, "dur": 4.158,
    "args": {
      "External id": 2471463,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858236.391, "dur": 1.063,
    "args": {
      "External id": 2471464,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858240.091, "dur": 0.932,
    "args": {
      "External id": 2471465,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858243.097, "dur": 0.965,
    "args": {
      "External id": 2471466,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858245.663, "dur": 1.173,
    "args": {
      "External id": 2471467,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858248.524, "dur": 3.121,
    "args": {
      "External id": 2471468,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858253.985, "dur": 0.871,
    "args": {
      "External id": 2471469,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858256.360, "dur": 0.939,
    "args": {
      "External id": 2471470,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858258.903, "dur": 0.740,
    "args": {
      "External id": 2471471,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858261.532, "dur": 1.056,
    "args": {
      "External id": 2471472,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858282.940, "dur": 144.863,
    "args": {
      "External id": 2471473,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858298.451, "dur": 125.002,
    "args": {
      "External id": 2471474,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858316.488, "dur": 12.174,
    "args": {
      "External id": 2471475,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858331.265, "dur": 64.408,
    "args": {
      "External id": 2471476,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858333.797, "dur": 61.571,
    "args": {
      "External id": 2471477,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858337.930, "dur": 6.322,
    "args": {
      "External id": 2471478,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858345.803, "dur": 48.953,
    "args": {
      "External id": 2471479,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5174
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858545.618, "dur": 582.513,
    "args": {
      "External id": 2471480,"Record function id": 0, "Ev Idx": 5175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858564.108, "dur": 552.300,
    "args": {
      "External id": 2471481,"Record function id": 0, "Ev Idx": 5176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858622.478, "dur": 5.645,
    "args": {
      "External id": 2471482,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858647.446, "dur": 30.359,
    "args": {
      "External id": 2471483,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858652.499, "dur": 3.553,
    "args": {
      "External id": 2471484,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858658.247, "dur": 0.575,
    "args": {
      "External id": 2471485,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858660.051, "dur": 0.355,
    "args": {
      "External id": 2471486,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858662.651, "dur": 0.374,
    "args": {
      "External id": 2471487,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858664.485, "dur": 0.357,
    "args": {
      "External id": 2471488,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858666.180, "dur": 0.591,
    "args": {
      "External id": 2471489,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858669.020, "dur": 0.536,
    "args": {
      "External id": 2471490,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858670.911, "dur": 0.466,
    "args": {
      "External id": 2471491,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858672.551, "dur": 2.160,
    "args": {
      "External id": 2471492,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858688.962, "dur": 34.344,
    "args": {
      "External id": 2471493,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858753.331, "dur": 100.141,
    "args": {
      "External id": 2471494,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858762.630, "dur": 3.059,
    "args": {
      "External id": 2471495,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858770.477, "dur": 9.399,
    "args": {
      "External id": 2471496,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858774.314, "dur": 5.145,
    "args": {
      "External id": 2471497,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858777.737, "dur": 0.550,
    "args": {
      "External id": 2471498,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858786.720, "dur": 25.888,
    "args": {
      "External id": 2471499,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858788.709, "dur": 0.436,
    "args": {
      "External id": 2471500,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858790.799, "dur": 0.543,
    "args": {
      "External id": 2471501,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858792.702, "dur": 0.997,
    "args": {
      "External id": 2471502,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858795.375, "dur": 0.460,
    "args": {
      "External id": 2471503,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858797.128, "dur": 0.444,
    "args": {
      "External id": 2471504,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858799.578, "dur": 2.143,
    "args": {
      "External id": 2471505,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858802.969, "dur": 0.259,
    "args": {
      "External id": 2471506,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858804.712, "dur": 0.471,
    "args": {
      "External id": 2471507,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858806.499, "dur": 0.486,
    "args": {
      "External id": 2471508,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858824.937, "dur": 21.572,
    "args": {
      "External id": 2471509,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858894.016, "dur": 151.617,
    "args": {
      "External id": 2471510,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858920.899, "dur": 120.889,
    "args": {
      "External id": 2471511,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5206, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657858929.819, "dur": 107.591,
    "args": {
      "External id": 2471512,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859063.592, "dur": 2.042,
    "args": {
      "External id": 2471513,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5208, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859142.210, "dur": 1622.272,
    "args": {
      "External id": 2471514,"Sequence number": 24740428, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5209
    }
  },
  {
    "ph": "f", "id": 220, "pid": 1336756, "tid": 1381189, "ts": 1514657859142.210,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859251.229, "dur": 106.953,
    "args": {
      "External id": 2471515,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859400.868, "dur": 40.982,
    "args": {
      "External id": 2471516,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859457.617, "dur": 51.091,
    "args": {
      "External id": 2471517,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859518.200, "dur": 52.811,
    "args": {
      "External id": 2471518,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859580.329, "dur": 48.362,
    "args": {
      "External id": 2471519,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859637.268, "dur": 26.691,
    "args": {
      "External id": 2471520,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859671.255, "dur": 43.329,
    "args": {
      "External id": 2471521,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859738.445, "dur": 24.724,
    "args": {
      "External id": 2471522,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859781.618, "dur": 31.658,
    "args": {
      "External id": 2471523,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859833.379, "dur": 19.467,
    "args": {
      "External id": 2471524,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859867.500, "dur": 16.432,
    "args": {
      "External id": 2471525,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859893.776, "dur": 30.868,
    "args": {
      "External id": 2471526,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657859927.665, "dur": 34.201,
    "args": {
      "External id": 2471527,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860024.350, "dur": 177.553,
    "args": {
      "External id": 2471528,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860103.815, "dur": 7.293,
    "args": {
      "External id": 2471529,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860113.060, "dur": 2.773,
    "args": {
      "External id": 2471530,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860237.167, "dur": 26.460,
    "args": {
      "External id": 2471531,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860275.306, "dur": 15.508,
    "args": {
      "External id": 2471532,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860300.411, "dur": 41.345,
    "args": {
      "External id": 2471533,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860347.245, "dur": 36.582,
    "args": {
      "External id": 2471534,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860389.990, "dur": 21.289,
    "args": {
      "External id": 2471535,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860415.350, "dur": 29.158,
    "args": {
      "External id": 2471536,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860451.429, "dur": 20.309,
    "args": {
      "External id": 2471537,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860477.884, "dur": 29.660,
    "args": {
      "External id": 2471538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860559.338, "dur": 31.511,
    "args": {
      "External id": 2471539,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860611.525, "dur": 23.255,
    "args": {
      "External id": 2471540,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860649.214, "dur": 20.770,
    "args": {
      "External id": 2471541,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860687.810, "dur": 15.099,
    "args": {
      "External id": 2471542,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860718.750, "dur": 17.446,
    "args": {
      "External id": 2471543,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860807.193, "dur": 16.212,
    "args": {
      "External id": 2471544,"Record function id": 0, "Ev Idx": 5239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860810.201, "dur": 12.318,
    "args": {
      "External id": 2471545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860814.318, "dur": 7.383,
    "args": {
      "External id": 2471546,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860815.888, "dur": 5.726,
    "args": {
      "External id": 2471547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860827.079, "dur": 4.573,
    "args": {
      "External id": 2471548,"Record function id": 0, "Ev Idx": 5243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860828.732, "dur": 2.487,
    "args": {
      "External id": 2471549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860829.394, "dur": 1.338,
    "args": {
      "External id": 2471550,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860829.898, "dur": 0.709,
    "args": {
      "External id": 2471551,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860834.866, "dur": 4.679,
    "args": {
      "External id": 2471552,"Record function id": 0, "Ev Idx": 5247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860836.339, "dur": 2.777,
    "args": {
      "External id": 2471553,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860837.228, "dur": 1.455,
    "args": {
      "External id": 2471554,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860837.765, "dur": 0.822,
    "args": {
      "External id": 2471555,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860842.640, "dur": 4.148,
    "args": {
      "External id": 2471556,"Record function id": 0, "Ev Idx": 5251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860844.006, "dur": 2.372,
    "args": {
      "External id": 2471557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860844.760, "dur": 1.213,
    "args": {
      "External id": 2471558,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860845.270, "dur": 0.617,
    "args": {
      "External id": 2471559,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860849.825, "dur": 3.832,
    "args": {
      "External id": 2471560,"Record function id": 0, "Ev Idx": 5255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860851.166, "dur": 2.096,
    "args": {
      "External id": 2471561,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860851.791, "dur": 1.041,
    "args": {
      "External id": 2471562,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860852.213, "dur": 0.555,
    "args": {
      "External id": 2471563,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860856.622, "dur": 3.346,
    "args": {
      "External id": 2471564,"Record function id": 0, "Ev Idx": 5259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860857.630, "dur": 1.932,
    "args": {
      "External id": 2471565,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860858.058, "dur": 1.097,
    "args": {
      "External id": 2471566,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860858.429, "dur": 0.653,
    "args": {
      "External id": 2471567,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860863.112, "dur": 4.616,
    "args": {
      "External id": 2471568,"Record function id": 0, "Ev Idx": 5263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860864.286, "dur": 3.044,
    "args": {
      "External id": 2471569,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860864.719, "dur": 2.211,
    "args": {
      "External id": 2471570,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860866.280, "dur": 0.552,
    "args": {
      "External id": 2471571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860870.801, "dur": 8.083,
    "args": {
      "External id": 2471572,"Record function id": 0, "Ev Idx": 5267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860871.858, "dur": 6.608,
    "args": {
      "External id": 2471573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860875.494, "dur": 2.536,
    "args": {
      "External id": 2471574,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860875.795, "dur": 2.162,
    "args": {
      "External id": 2471575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860881.966, "dur": 4.089,
    "args": {
      "External id": 2471576,"Record function id": 0, "Ev Idx": 5271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860883.349, "dur": 2.313,
    "args": {
      "External id": 2471577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860883.799, "dur": 1.443,
    "args": {
      "External id": 2471578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860884.291, "dur": 0.858,
    "args": {
      "External id": 2471579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860889.444, "dur": 36660.770,
    "args": {
      "External id": 2471580,"Record function id": 0, "Sequence number": 24740427, "Fwd thread id": 1, "Ev Idx": 5275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860893.888, "dur": 36631.592,
    "args": {
      "External id": 2471581,"Sequence number": 24740427, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5276
    }
  },
  {
    "ph": "f", "id": 221, "pid": 1336756, "tid": 1381189, "ts": 1514657860893.888,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.3)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860920.867, "dur": 40.814,
    "args": {
      "External id": 2471582,"Record function id": 0, "Ev Idx": 5277
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.3)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657860968.995, "dur": 104.081,
    "args": {
      "External id": 2471583,"Record function id": 0, "Ev Idx": 5278
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.3)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861081.306, "dur": 36436.459,
    "args": {
      "External id": 2471584,"Record function id": 0, "Ev Idx": 5279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861172.310, "dur": 6.758,
    "args": {
      "External id": 2471585,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861189.301, "dur": 8.467,
    "args": {
      "External id": 2471586,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861211.506, "dur": 35507.592,
    "args": {
      "External id": 2471587,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861224.412, "dur": 35486.708,
    "args": {
      "External id": 2471588,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861265.736, "dur": 13.487,
    "args": {
      "External id": 2471589,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861285.243, "dur": 35389.353,
    "args": {
      "External id": 2471590,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861287.697, "dur": 35386.176,
    "args": {
      "External id": 2471591,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861291.460, "dur": 5.422,
    "args": {
      "External id": 2471592,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657861300.146, "dur": 35370.201,
    "args": {
      "External id": 2471593,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657896802.925, "dur": 8.476,
    "args": {
      "External id": 2471594,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657896805.829, "dur": 5.248,
    "args": {
      "External id": 2471595,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657896840.773, "dur": 390.102,
    "args": {
      "External id": 2471596,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657896870.286, "dur": 355.471,
    "args": {
      "External id": 2471597,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5292, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657896880.903, "dur": 338.775,
    "args": {
      "External id": 2471598,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897254.911, "dur": 2.394,
    "args": {
      "External id": 2471599,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5294, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897316.684, "dur": 6.438,
    "args": {
      "External id": 2471600,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897366.573, "dur": 1.199,
    "args": {
      "External id": 2471601,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897384.564, "dur": 3.403,
    "args": {
      "External id": 2471602,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897399.404, "dur": 1.004,
    "args": {
      "External id": 2471603,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897410.991, "dur": 1.014,
    "args": {
      "External id": 2471604,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897421.299, "dur": 1.002,
    "args": {
      "External id": 2471605,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897432.991, "dur": 2.832,
    "args": {
      "External id": 2471606,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897450.081, "dur": 1.232,
    "args": {
      "External id": 2471607,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897460.808, "dur": 0.941,
    "args": {
      "External id": 2471608,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897567.429, "dur": 2746.431,
    "args": {
      "External id": 2471609,"Record function id": 0, "Ev Idx": 5304
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.2)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897588.708, "dur": 1038.525,
    "args": {
      "External id": 2471610,"Record function id": 0, "Ev Idx": 5305
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897614.141, "dur": 312.461,
    "args": {
      "External id": 2471611,"Record function id": 0, "Ev Idx": 5306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897697.624, "dur": 4.625,
    "args": {
      "External id": 2471612,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897705.244, "dur": 1.326,
    "args": {
      "External id": 2471613,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897709.709, "dur": 0.739,
    "args": {
      "External id": 2471614,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897712.042, "dur": 2.544,
    "args": {
      "External id": 2471615,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897716.159, "dur": 1.237,
    "args": {
      "External id": 2471616,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897718.937, "dur": 1.031,
    "args": {
      "External id": 2471617,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897722.652, "dur": 0.871,
    "args": {
      "External id": 2471618,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897725.362, "dur": 0.939,
    "args": {
      "External id": 2471619,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897727.559, "dur": 1.077,
    "args": {
      "External id": 2471620,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897730.681, "dur": 0.805,
    "args": {
      "External id": 2471621,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897751.556, "dur": 147.635,
    "args": {
      "External id": 2471622,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897767.324, "dur": 127.632,
    "args": {
      "External id": 2471623,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897783.829, "dur": 12.641,
    "args": {
      "External id": 2471624,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897799.500, "dur": 67.268,
    "args": {
      "External id": 2471625,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897802.079, "dur": 64.366,
    "args": {
      "External id": 2471626,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897805.920, "dur": 8.418,
    "args": {
      "External id": 2471627,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657897816.153, "dur": 49.661,
    "args": {
      "External id": 2471628,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5323
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898058.023, "dur": 561.610,
    "args": {
      "External id": 2471629,"Record function id": 0, "Ev Idx": 5324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898075.245, "dur": 532.418,
    "args": {
      "External id": 2471630,"Record function id": 0, "Ev Idx": 5325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898138.975, "dur": 5.514,
    "args": {
      "External id": 2471631,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898166.855, "dur": 27.725,
    "args": {
      "External id": 2471632,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898173.646, "dur": 2.669,
    "args": {
      "External id": 2471633,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898177.701, "dur": 0.424,
    "args": {
      "External id": 2471634,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898179.106, "dur": 0.361,
    "args": {
      "External id": 2471635,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898181.163, "dur": 0.293,
    "args": {
      "External id": 2471636,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898182.464, "dur": 0.521,
    "args": {
      "External id": 2471637,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898184.163, "dur": 0.518,
    "args": {
      "External id": 2471638,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898186.529, "dur": 2.317,
    "args": {
      "External id": 2471639,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898189.760, "dur": 0.289,
    "args": {
      "External id": 2471640,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898190.954, "dur": 0.791,
    "args": {
      "External id": 2471641,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898205.368, "dur": 34.185,
    "args": {
      "External id": 2471642,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898269.142, "dur": 97.442,
    "args": {
      "External id": 2471643,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898279.433, "dur": 3.495,
    "args": {
      "External id": 2471644,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898287.872, "dur": 9.993,
    "args": {
      "External id": 2471645,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898292.134, "dur": 5.317,
    "args": {
      "External id": 2471646,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898295.679, "dur": 0.577,
    "args": {
      "External id": 2471647,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898304.181, "dur": 24.757,
    "args": {
      "External id": 2471648,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898306.288, "dur": 0.596,
    "args": {
      "External id": 2471649,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898308.016, "dur": 0.799,
    "args": {
      "External id": 2471650,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898309.953, "dur": 1.007,
    "args": {
      "External id": 2471651,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898311.942, "dur": 2.696,
    "args": {
      "External id": 2471652,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898315.731, "dur": 0.355,
    "args": {
      "External id": 2471653,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898317.751, "dur": 0.315,
    "args": {
      "External id": 2471654,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898320.864, "dur": 0.422,
    "args": {
      "External id": 2471655,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898322.311, "dur": 0.359,
    "args": {
      "External id": 2471656,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898324.211, "dur": 0.359,
    "args": {
      "External id": 2471657,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898340.263, "dur": 19.569,
    "args": {
      "External id": 2471658,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898408.061, "dur": 113.362,
    "args": {
      "External id": 2471659,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898433.495, "dur": 84.585,
    "args": {
      "External id": 2471660,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5355, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898442.592, "dur": 71.522,
    "args": {
      "External id": 2471661,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898553.272, "dur": 2.786,
    "args": {
      "External id": 2471662,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5357, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898634.117, "dur": 1655.126,
    "args": {
      "External id": 2471663,"Sequence number": 24740426, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5358
    }
  },
  {
    "ph": "f", "id": 222, "pid": 1336756, "tid": 1381189, "ts": 1514657898634.117,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898746.881, "dur": 106.091,
    "args": {
      "External id": 2471664,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898894.885, "dur": 39.647,
    "args": {
      "External id": 2471665,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657898951.115, "dur": 92.795,
    "args": {
      "External id": 2471666,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899058.337, "dur": 37.617,
    "args": {
      "External id": 2471667,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899104.419, "dur": 45.848,
    "args": {
      "External id": 2471668,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899157.127, "dur": 26.954,
    "args": {
      "External id": 2471669,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899191.458, "dur": 41.830,
    "args": {
      "External id": 2471670,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899257.473, "dur": 23.801,
    "args": {
      "External id": 2471671,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899299.927, "dur": 33.266,
    "args": {
      "External id": 2471672,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899355.177, "dur": 19.000,
    "args": {
      "External id": 2471673,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899390.115, "dur": 16.733,
    "args": {
      "External id": 2471674,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899414.906, "dur": 28.838,
    "args": {
      "External id": 2471675,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899446.903, "dur": 32.649,
    "args": {
      "External id": 2471676,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899508.519, "dur": 189.428,
    "args": {
      "External id": 2471677,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899604.068, "dur": 6.902,
    "args": {
      "External id": 2471678,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899613.009, "dur": 2.631,
    "args": {
      "External id": 2471679,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899733.313, "dur": 26.384,
    "args": {
      "External id": 2471680,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899772.131, "dur": 15.681,
    "args": {
      "External id": 2471681,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899795.627, "dur": 42.045,
    "args": {
      "External id": 2471682,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899843.126, "dur": 35.702,
    "args": {
      "External id": 2471683,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899885.568, "dur": 22.547,
    "args": {
      "External id": 2471684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899914.511, "dur": 29.088,
    "args": {
      "External id": 2471685,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899949.372, "dur": 20.430,
    "args": {
      "External id": 2471686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657899977.081, "dur": 68.609,
    "args": {
      "External id": 2471687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900083.014, "dur": 30.463,
    "args": {
      "External id": 2471688,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900135.917, "dur": 27.492,
    "args": {
      "External id": 2471689,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900177.730, "dur": 17.361,
    "args": {
      "External id": 2471690,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900212.993, "dur": 14.351,
    "args": {
      "External id": 2471691,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900239.101, "dur": 20.057,
    "args": {
      "External id": 2471692,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900335.263, "dur": 15.211,
    "args": {
      "External id": 2471693,"Record function id": 0, "Ev Idx": 5388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900338.514, "dur": 10.998,
    "args": {
      "External id": 2471694,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900342.835, "dur": 5.878,
    "args": {
      "External id": 2471695,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900344.398, "dur": 4.224,
    "args": {
      "External id": 2471696,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900354.639, "dur": 4.934,
    "args": {
      "External id": 2471697,"Record function id": 0, "Ev Idx": 5392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900355.920, "dur": 3.192,
    "args": {
      "External id": 2471698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900356.989, "dur": 1.658,
    "args": {
      "External id": 2471699,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900357.653, "dur": 0.880,
    "args": {
      "External id": 2471700,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900362.754, "dur": 3.925,
    "args": {
      "External id": 2471701,"Record function id": 0, "Ev Idx": 5396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900363.825, "dur": 2.439,
    "args": {
      "External id": 2471702,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900364.456, "dur": 1.343,
    "args": {
      "External id": 2471703,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900364.978, "dur": 0.741,
    "args": {
      "External id": 2471704,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900369.746, "dur": 3.899,
    "args": {
      "External id": 2471705,"Record function id": 0, "Ev Idx": 5400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900371.095, "dur": 2.143,
    "args": {
      "External id": 2471706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900371.553, "dur": 1.252,
    "args": {
      "External id": 2471707,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900371.992, "dur": 0.739,
    "args": {
      "External id": 2471708,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900376.670, "dur": 3.388,
    "args": {
      "External id": 2471709,"Record function id": 0, "Ev Idx": 5404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900377.785, "dur": 1.838,
    "args": {
      "External id": 2471710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900378.252, "dur": 0.920,
    "args": {
      "External id": 2471711,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900378.556, "dur": 0.543,
    "args": {
      "External id": 2471712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900383.105, "dur": 4.504,
    "args": {
      "External id": 2471713,"Record function id": 0, "Ev Idx": 5408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900384.218, "dur": 2.953,
    "args": {
      "External id": 2471714,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900384.637, "dur": 2.053,
    "args": {
      "External id": 2471715,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900385.724, "dur": 0.869,
    "args": {
      "External id": 2471716,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900390.937, "dur": 5.960,
    "args": {
      "External id": 2471717,"Record function id": 0, "Ev Idx": 5412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900391.905, "dur": 4.595,
    "args": {
      "External id": 2471718,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900392.881, "dur": 3.174,
    "args": {
      "External id": 2471719,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900393.310, "dur": 2.677,
    "args": {
      "External id": 2471720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900400.091, "dur": 3.650,
    "args": {
      "External id": 2471721,"Record function id": 0, "Ev Idx": 5416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900401.254, "dur": 2.043,
    "args": {
      "External id": 2471722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900401.686, "dur": 1.206,
    "args": {
      "External id": 2471723,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900401.953, "dur": 0.857,
    "args": {
      "External id": 2471724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900406.728, "dur": 3.441,
    "args": {
      "External id": 2471725,"Record function id": 0, "Ev Idx": 5420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900407.759, "dur": 2.009,
    "args": {
      "External id": 2471726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900408.191, "dur": 1.148,
    "args": {
      "External id": 2471727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900408.674, "dur": 0.599,
    "args": {
      "External id": 2471728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900413.942, "dur": 36541.949,
    "args": {
      "External id": 2471729,"Record function id": 0, "Sequence number": 24740425, "Fwd thread id": 1, "Ev Idx": 5424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900415.489, "dur": 36532.076,
    "args": {
      "External id": 2471730,"Sequence number": 24740425, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5425
    }
  },
  {
    "ph": "f", "id": 223, "pid": 1336756, "tid": 1381189, "ts": 1514657900415.489,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.2)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900444.194, "dur": 36.059,
    "args": {
      "External id": 2471731,"Record function id": 0, "Ev Idx": 5426
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.2)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900487.574, "dur": 86.604,
    "args": {
      "External id": 2471732,"Record function id": 0, "Ev Idx": 5427
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.2)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900581.855, "dur": 36358.251,
    "args": {
      "External id": 2471733,"Record function id": 0, "Ev Idx": 5428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900676.146, "dur": 6.611,
    "args": {
      "External id": 2471734,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900692.689, "dur": 5.142,
    "args": {
      "External id": 2471735,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900715.962, "dur": 35390.263,
    "args": {
      "External id": 2471736,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900729.321, "dur": 35368.137,
    "args": {
      "External id": 2471737,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900775.185, "dur": 13.504,
    "args": {
      "External id": 2471738,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900794.866, "dur": 35261.190,
    "args": {
      "External id": 2471739,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900797.212, "dur": 35258.206,
    "args": {
      "External id": 2471740,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900802.086, "dur": 5.759,
    "args": {
      "External id": 2471741,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657900810.755, "dur": 35240.949,
    "args": {
      "External id": 2471742,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936194.573, "dur": 8.407,
    "args": {
      "External id": 2471743,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936197.316, "dur": 5.377,
    "args": {
      "External id": 2471744,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936234.163, "dur": 413.914,
    "args": {
      "External id": 2471745,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936264.491, "dur": 378.782,
    "args": {
      "External id": 2471746,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5441, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936275.604, "dur": 362.030,
    "args": {
      "External id": 2471747,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936671.425, "dur": 2.258,
    "args": {
      "External id": 2471748,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5443, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936734.527, "dur": 6.824,
    "args": {
      "External id": 2471749,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936786.179, "dur": 2.793,
    "args": {
      "External id": 2471750,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936804.726, "dur": 1.279,
    "args": {
      "External id": 2471751,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936817.535, "dur": 0.889,
    "args": {
      "External id": 2471752,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936831.050, "dur": 0.984,
    "args": {
      "External id": 2471753,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936842.890, "dur": 3.071,
    "args": {
      "External id": 2471754,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936856.891, "dur": 1.011,
    "args": {
      "External id": 2471755,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936868.342, "dur": 1.215,
    "args": {
      "External id": 2471756,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936880.270, "dur": 0.991,
    "args": {
      "External id": 2471757,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657936969.749, "dur": 2768.234,
    "args": {
      "External id": 2471758,"Record function id": 0, "Ev Idx": 5453
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.1)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937026.606, "dur": 1028.971,
    "args": {
      "External id": 2471759,"Record function id": 0, "Ev Idx": 5454
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937042.225, "dur": 316.136,
    "args": {
      "External id": 2471760,"Record function id": 0, "Ev Idx": 5455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937129.380, "dur": 4.635,
    "args": {
      "External id": 2471761,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937137.673, "dur": 3.067,
    "args": {
      "External id": 2471762,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937142.850, "dur": 1.423,
    "args": {
      "External id": 2471763,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937146.047, "dur": 0.996,
    "args": {
      "External id": 2471764,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937148.615, "dur": 0.602,
    "args": {
      "External id": 2471765,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937152.041, "dur": 1.180,
    "args": {
      "External id": 2471766,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937155.069, "dur": 1.058,
    "args": {
      "External id": 2471767,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937157.862, "dur": 0.756,
    "args": {
      "External id": 2471768,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937160.441, "dur": 1.000,
    "args": {
      "External id": 2471769,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937164.076, "dur": 2.971,
    "args": {
      "External id": 2471770,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937186.060, "dur": 143.746,
    "args": {
      "External id": 2471771,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937201.585, "dur": 123.643,
    "args": {
      "External id": 2471772,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937218.178, "dur": 11.637,
    "args": {
      "External id": 2471773,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937233.000, "dur": 64.028,
    "args": {
      "External id": 2471774,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937235.571, "dur": 61.205,
    "args": {
      "External id": 2471775,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937239.677, "dur": 6.186,
    "args": {
      "External id": 2471776,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937247.705, "dur": 48.501,
    "args": {
      "External id": 2471777,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5472
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937454.564, "dur": 593.298,
    "args": {
      "External id": 2471778,"Record function id": 0, "Ev Idx": 5473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937469.842, "dur": 564.799,
    "args": {
      "External id": 2471779,"Record function id": 0, "Ev Idx": 5474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937552.398, "dur": 6.250,
    "args": {
      "External id": 2471780,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937574.511, "dur": 28.022,
    "args": {
      "External id": 2471781,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937579.742, "dur": 1.514,
    "args": {
      "External id": 2471782,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937583.346, "dur": 0.825,
    "args": {
      "External id": 2471783,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937585.440, "dur": 0.728,
    "args": {
      "External id": 2471784,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937588.071, "dur": 0.544,
    "args": {
      "External id": 2471785,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937589.853, "dur": 2.198,
    "args": {
      "External id": 2471786,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937593.522, "dur": 0.689,
    "args": {
      "External id": 2471787,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937595.853, "dur": 0.293,
    "args": {
      "External id": 2471788,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937597.713, "dur": 0.410,
    "args": {
      "External id": 2471789,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937599.305, "dur": 0.285,
    "args": {
      "External id": 2471790,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937614.121, "dur": 35.431,
    "args": {
      "External id": 2471791,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937680.327, "dur": 97.798,
    "args": {
      "External id": 2471792,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937692.301, "dur": 2.855,
    "args": {
      "External id": 2471793,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937699.981, "dur": 10.989,
    "args": {
      "External id": 2471794,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937705.423, "dur": 5.127,
    "args": {
      "External id": 2471795,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937708.908, "dur": 0.482,
    "args": {
      "External id": 2471796,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937717.709, "dur": 22.223,
    "args": {
      "External id": 2471797,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937719.512, "dur": 0.348,
    "args": {
      "External id": 2471798,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937721.183, "dur": 2.194,
    "args": {
      "External id": 2471799,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937724.654, "dur": 0.533,
    "args": {
      "External id": 2471800,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937727.008, "dur": 0.572,
    "args": {
      "External id": 2471801,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937728.560, "dur": 0.412,
    "args": {
      "External id": 2471802,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937730.080, "dur": 0.443,
    "args": {
      "External id": 2471803,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937731.912, "dur": 0.607,
    "args": {
      "External id": 2471804,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937733.592, "dur": 0.294,
    "args": {
      "External id": 2471805,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937735.206, "dur": 0.433,
    "args": {
      "External id": 2471806,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937749.499, "dur": 19.444,
    "args": {
      "External id": 2471807,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937821.772, "dur": 109.730,
    "args": {
      "External id": 2471808,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937845.206, "dur": 82.921,
    "args": {
      "External id": 2471809,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5504, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937854.506, "dur": 68.867,
    "args": {
      "External id": 2471810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657937944.242, "dur": 2.253,
    "args": {
      "External id": 2471811,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5506, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938063.671, "dur": 1655.011,
    "args": {
      "External id": 2471812,"Sequence number": 24740424, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5507
    }
  },
  {
    "ph": "f", "id": 224, "pid": 1336756, "tid": 1381189, "ts": 1514657938063.671,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938177.531, "dur": 105.869,
    "args": {
      "External id": 2471813,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938324.439, "dur": 37.017,
    "args": {
      "External id": 2471814,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938376.899, "dur": 49.291,
    "args": {
      "External id": 2471815,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938437.814, "dur": 33.120,
    "args": {
      "External id": 2471816,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938477.663, "dur": 45.499,
    "args": {
      "External id": 2471817,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938548.027, "dur": 33.307,
    "args": {
      "External id": 2471818,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938590.284, "dur": 42.889,
    "args": {
      "External id": 2471819,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938659.596, "dur": 26.018,
    "args": {
      "External id": 2471820,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938703.654, "dur": 32.553,
    "args": {
      "External id": 2471821,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938756.522, "dur": 20.176,
    "args": {
      "External id": 2471822,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938789.809, "dur": 16.447,
    "args": {
      "External id": 2471823,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938814.372, "dur": 29.822,
    "args": {
      "External id": 2471824,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938847.629, "dur": 33.936,
    "args": {
      "External id": 2471825,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657938911.440, "dur": 229.393,
    "args": {
      "External id": 2471826,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939040.689, "dur": 6.927,
    "args": {
      "External id": 2471827,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939049.773, "dur": 2.392,
    "args": {
      "External id": 2471828,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939174.921, "dur": 25.059,
    "args": {
      "External id": 2471829,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939211.476, "dur": 16.872,
    "args": {
      "External id": 2471830,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939237.051, "dur": 42.969,
    "args": {
      "External id": 2471831,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939287.086, "dur": 37.422,
    "args": {
      "External id": 2471832,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939333.095, "dur": 21.962,
    "args": {
      "External id": 2471833,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939360.495, "dur": 31.004,
    "args": {
      "External id": 2471834,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939397.799, "dur": 21.128,
    "args": {
      "External id": 2471835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939426.158, "dur": 47.983,
    "args": {
      "External id": 2471836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939496.233, "dur": 22.741,
    "args": {
      "External id": 2471837,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939557.511, "dur": 32.181,
    "args": {
      "External id": 2471838,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939606.406, "dur": 18.156,
    "args": {
      "External id": 2471839,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939643.948, "dur": 16.396,
    "args": {
      "External id": 2471840,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939673.100, "dur": 17.806,
    "args": {
      "External id": 2471841,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939760.187, "dur": 16.275,
    "args": {
      "External id": 2471842,"Record function id": 0, "Ev Idx": 5537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939763.952, "dur": 11.582,
    "args": {
      "External id": 2471843,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939768.214, "dur": 6.399,
    "args": {
      "External id": 2471844,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939770.361, "dur": 4.128,
    "args": {
      "External id": 2471845,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939780.463, "dur": 5.638,
    "args": {
      "External id": 2471846,"Record function id": 0, "Ev Idx": 5541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939781.819, "dur": 3.813,
    "args": {
      "External id": 2471847,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939783.232, "dur": 1.913,
    "args": {
      "External id": 2471848,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939784.099, "dur": 0.971,
    "args": {
      "External id": 2471849,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939789.300, "dur": 5.806,
    "args": {
      "External id": 2471850,"Record function id": 0, "Ev Idx": 5545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939790.899, "dur": 3.765,
    "args": {
      "External id": 2471851,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939792.127, "dur": 2.080,
    "args": {
      "External id": 2471852,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939792.857, "dur": 1.284,
    "args": {
      "External id": 2471853,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939798.337, "dur": 4.477,
    "args": {
      "External id": 2471854,"Record function id": 0, "Ev Idx": 5549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939799.831, "dur": 2.537,
    "args": {
      "External id": 2471855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939800.752, "dur": 1.186,
    "args": {
      "External id": 2471856,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939801.249, "dur": 0.621,
    "args": {
      "External id": 2471857,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939806.230, "dur": 3.729,
    "args": {
      "External id": 2471858,"Record function id": 0, "Ev Idx": 5553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939807.240, "dur": 2.312,
    "args": {
      "External id": 2471859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939807.710, "dur": 1.302,
    "args": {
      "External id": 2471860,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939808.079, "dur": 0.861,
    "args": {
      "External id": 2471861,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939813.038, "dur": 4.692,
    "args": {
      "External id": 2471862,"Record function id": 0, "Ev Idx": 5557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939814.392, "dur": 2.889,
    "args": {
      "External id": 2471863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939815.419, "dur": 1.373,
    "args": {
      "External id": 2471864,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939815.820, "dur": 0.844,
    "args": {
      "External id": 2471865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939821.171, "dur": 5.192,
    "args": {
      "External id": 2471866,"Record function id": 0, "Ev Idx": 5561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939822.107, "dur": 3.779,
    "args": {
      "External id": 2471867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939822.822, "dur": 2.608,
    "args": {
      "External id": 2471868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939823.269, "dur": 2.068,
    "args": {
      "External id": 2471869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939829.417, "dur": 4.079,
    "args": {
      "External id": 2471870,"Record function id": 0, "Ev Idx": 5565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939830.550, "dur": 2.524,
    "args": {
      "External id": 2471871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939831.463, "dur": 1.178,
    "args": {
      "External id": 2471872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939831.732, "dur": 0.803,
    "args": {
      "External id": 2471873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939836.483, "dur": 4.529,
    "args": {
      "External id": 2471874,"Record function id": 0, "Ev Idx": 5569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939837.570, "dur": 2.985,
    "args": {
      "External id": 2471875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939838.224, "dur": 1.907,
    "args": {
      "External id": 2471876,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939839.148, "dur": 0.874,
    "args": {
      "External id": 2471877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939844.569, "dur": 36656.117,
    "args": {
      "External id": 2471878,"Record function id": 0, "Sequence number": 24740423, "Fwd thread id": 1, "Ev Idx": 5573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939846.226, "dur": 36646.218,
    "args": {
      "External id": 2471879,"Sequence number": 24740423, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5574
    }
  },
  {
    "ph": "f", "id": 225, "pid": 1336756, "tid": 1381189, "ts": 1514657939846.226,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.1)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939872.930, "dur": 37.203,
    "args": {
      "External id": 2471880,"Record function id": 0, "Ev Idx": 5575
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.1)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657939917.472, "dur": 98.802,
    "args": {
      "External id": 2471881,"Record function id": 0, "Ev Idx": 5576
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.1)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940025.227, "dur": 36459.385,
    "args": {
      "External id": 2471882,"Record function id": 0, "Ev Idx": 5577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940112.989, "dur": 6.767,
    "args": {
      "External id": 2471883,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940129.669, "dur": 5.150,
    "args": {
      "External id": 2471884,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940148.861, "dur": 35567.374,
    "args": {
      "External id": 2471885,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940161.654, "dur": 35545.704,
    "args": {
      "External id": 2471886,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940207.629, "dur": 13.769,
    "args": {
      "External id": 2471887,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940227.510, "dur": 35442.950,
    "args": {
      "External id": 2471888,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940230.003, "dur": 35439.778,
    "args": {
      "External id": 2471889,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940233.748, "dur": 8.514,
    "args": {
      "External id": 2471890,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657940244.076, "dur": 35421.860,
    "args": {
      "External id": 2471891,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657975803.690, "dur": 8.655,
    "args": {
      "External id": 2471892,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657975806.606, "dur": 5.424,
    "args": {
      "External id": 2471893,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657975844.576, "dur": 361.388,
    "args": {
      "External id": 2471894,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657975870.835, "dur": 330.095,
    "args": {
      "External id": 2471895,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5590, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514657975883.196, "dur": 312.083,
    "args": {
      "External id": 2471896,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976225.831, "dur": 2.189,
    "args": {
      "External id": 2471897,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5592, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976286.637, "dur": 6.691,
    "args": {
      "External id": 2471898,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976337.380, "dur": 2.392,
    "args": {
      "External id": 2471899,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976355.243, "dur": 1.106,
    "args": {
      "External id": 2471900,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976367.078, "dur": 0.773,
    "args": {
      "External id": 2471901,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976378.224, "dur": 1.270,
    "args": {
      "External id": 2471902,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976388.840, "dur": 2.769,
    "args": {
      "External id": 2471903,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976402.048, "dur": 1.088,
    "args": {
      "External id": 2471904,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976414.534, "dur": 1.261,
    "args": {
      "External id": 2471905,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976424.946, "dur": 0.795,
    "args": {
      "External id": 2471906,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976514.570, "dur": 2199.688,
    "args": {
      "External id": 2471907,"Record function id": 0, "Ev Idx": 5602
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.0)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976553.857, "dur": 489.897,
    "args": {
      "External id": 2471908,"Record function id": 0, "Ev Idx": 5603
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976570.139, "dur": 321.208,
    "args": {
      "External id": 2471909,"Record function id": 0, "Ev Idx": 5604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976664.157, "dur": 4.803,
    "args": {
      "External id": 2471910,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976672.250, "dur": 1.035,
    "args": {
      "External id": 2471911,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976674.989, "dur": 1.641,
    "args": {
      "External id": 2471912,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976678.478, "dur": 0.916,
    "args": {
      "External id": 2471913,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976680.949, "dur": 1.270,
    "args": {
      "External id": 2471914,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976683.555, "dur": 1.328,
    "args": {
      "External id": 2471915,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976686.447, "dur": 0.947,
    "args": {
      "External id": 2471916,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976690.747, "dur": 0.856,
    "args": {
      "External id": 2471917,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976693.203, "dur": 1.008,
    "args": {
      "External id": 2471918,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976696.102, "dur": 0.873,
    "args": {
      "External id": 2471919,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976716.014, "dur": 147.118,
    "args": {
      "External id": 2471920,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976733.673, "dur": 124.942,
    "args": {
      "External id": 2471921,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976748.219, "dur": 12.917,
    "args": {
      "External id": 2471922,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976764.906, "dur": 64.280,
    "args": {
      "External id": 2471923,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976767.501, "dur": 61.378,
    "args": {
      "External id": 2471924,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976771.813, "dur": 5.516,
    "args": {
      "External id": 2471925,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657976779.244, "dur": 49.004,
    "args": {
      "External id": 2471926,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977052.824, "dur": 1636.534,
    "args": {
      "External id": 2471927,"Sequence number": 24740422, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5622
    }
  },
  {
    "ph": "f", "id": 226, "pid": 1336756, "tid": 1381189, "ts": 1514657977052.824,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977162.499, "dur": 108.310,
    "args": {
      "External id": 2471928,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977310.097, "dur": 41.468,
    "args": {
      "External id": 2471929,"kernel_hash": "c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/2z/c2zszzsomz26dv2qwvmiurvgyrl5qxtyn6ocoqkdpzzhfzwqp5q6.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977367.960, "dur": 50.423,
    "args": {
      "External id": 2471930,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977429.702, "dur": 33.070,
    "args": {
      "External id": 2471931,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977469.697, "dur": 46.956,
    "args": {
      "External id": 2471932,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977524.764, "dur": 48.713,
    "args": {
      "External id": 2471933,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977583.776, "dur": 46.259,
    "args": {
      "External id": 2471934,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977653.143, "dur": 24.288,
    "args": {
      "External id": 2471935,"kernel_hash": "cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jd/cjd6g3jfgntyoifzno6l2urmnhif7alxtw6zz7miov66vywxwv7d.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977695.192, "dur": 32.041,
    "args": {
      "External id": 2471936,"kernel_hash": "cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/xh/cxhg545cyph4gbk5vymzyh72vho7oogk3ycan2lp4cxntklp5hew.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977769.914, "dur": 21.921,
    "args": {
      "External id": 2471937,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977805.395, "dur": 16.162,
    "args": {
      "External id": 2471938,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977830.558, "dur": 30.929,
    "args": {
      "External id": 2471939,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977864.792, "dur": 35.763,
    "args": {
      "External id": 2471940,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657977929.066, "dur": 210.920,
    "args": {
      "External id": 2471941,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978040.733, "dur": 6.431,
    "args": {
      "External id": 2471942,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978049.209, "dur": 2.237,
    "args": {
      "External id": 2471943,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978175.749, "dur": 28.676,
    "args": {
      "External id": 2471944,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978216.417, "dur": 17.396,
    "args": {
      "External id": 2471945,"kernel_hash": "cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/b6/cb6oumcqb462ndf2ivcd2sc6qg6mfjahptjw4ih7cesqhfpziv3b.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978243.315, "dur": 44.676,
    "args": {
      "External id": 2471946,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978294.723, "dur": 38.158,
    "args": {
      "External id": 2471947,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978341.202, "dur": 22.014,
    "args": {
      "External id": 2471948,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978368.778, "dur": 31.104,
    "args": {
      "External id": 2471949,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978406.100, "dur": 21.214,
    "args": {
      "External id": 2471950,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978434.530, "dur": 31.287,
    "args": {
      "External id": 2471951,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978482.291, "dur": 22.183,
    "args": {
      "External id": 2471952,"kernel_hash": "c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/7p/c7pqhdjojzeazwsh5g3ibnyk4m2nfh3q73e3zc7hljz4yvhe7c4h.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978520.479, "dur": 44.053,
    "args": {
      "External id": 2471953,"kernel_hash": "cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/vg/cvgvwkysjpcw75q5t3guce5bep2so25f7i5zxc6ipt5f56shof73.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978582.046, "dur": 19.686,
    "args": {
      "External id": 2471954,"kernel_hash": "cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/u4/cu42i6zmxtvs2i7cs6x4ixaexke3lhgdiaye5oxwm7s3u6em6co4.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978616.307, "dur": 15.689,
    "args": {
      "External id": 2471955,"kernel_hash": "ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/ki/ckihofrorhmzazt2wfn2dz2m5wfx3auisu2gvfo2jsepr4zyekmq.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978643.604, "dur": 18.354,
    "args": {
      "External id": 2471956,"kernel_hash": "cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ck/cckrpn5uuxyvbancoio3sz2onrjto236n54xx4ycphq5p6r3jlfj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978736.567, "dur": 15.184,
    "args": {
      "External id": 2471957,"Record function id": 0, "Ev Idx": 5652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978740.098, "dur": 10.757,
    "args": {
      "External id": 2471958,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978744.117, "dur": 5.855,
    "args": {
      "External id": 2471959,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978745.654, "dur": 4.175,
    "args": {
      "External id": 2471960,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978755.381, "dur": 4.900,
    "args": {
      "External id": 2471961,"Record function id": 0, "Ev Idx": 5656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978756.738, "dur": 3.089,
    "args": {
      "External id": 2471962,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978757.513, "dur": 1.695,
    "args": {
      "External id": 2471963,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978758.165, "dur": 0.927,
    "args": {
      "External id": 2471964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978763.520, "dur": 4.565,
    "args": {
      "External id": 2471965,"Record function id": 0, "Ev Idx": 5660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978765.162, "dur": 2.518,
    "args": {
      "External id": 2471966,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978765.785, "dur": 1.435,
    "args": {
      "External id": 2471967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978766.183, "dur": 0.964,
    "args": {
      "External id": 2471968,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978771.106, "dur": 4.221,
    "args": {
      "External id": 2471969,"Record function id": 0, "Ev Idx": 5664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978772.602, "dur": 2.314,
    "args": {
      "External id": 2471970,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978773.363, "dur": 1.147,
    "args": {
      "External id": 2471971,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978773.746, "dur": 0.691,
    "args": {
      "External id": 2471972,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978778.347, "dur": 3.901,
    "args": {
      "External id": 2471973,"Record function id": 0, "Ev Idx": 5668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978779.864, "dur": 1.975,
    "args": {
      "External id": 2471974,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978780.469, "dur": 0.959,
    "args": {
      "External id": 2471975,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978780.733, "dur": 0.627,
    "args": {
      "External id": 2471976,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978785.354, "dur": 4.428,
    "args": {
      "External id": 2471977,"Record function id": 0, "Ev Idx": 5672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978787.078, "dur": 2.219,
    "args": {
      "External id": 2471978,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978787.514, "dur": 1.344,
    "args": {
      "External id": 2471979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978788.120, "dur": 0.630,
    "args": {
      "External id": 2471980,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978793.112, "dur": 5.189,
    "args": {
      "External id": 2471981,"Record function id": 0, "Ev Idx": 5676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978794.464, "dur": 3.358,
    "args": {
      "External id": 2471982,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978795.029, "dur": 2.381,
    "args": {
      "External id": 2471983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978795.504, "dur": 1.823,
    "args": {
      "External id": 2471984,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978801.379, "dur": 4.261,
    "args": {
      "External id": 2471985,"Record function id": 0, "Ev Idx": 5680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978802.909, "dur": 2.309,
    "args": {
      "External id": 2471986,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978803.742, "dur": 1.067,
    "args": {
      "External id": 2471987,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978804.036, "dur": 0.668,
    "args": {
      "External id": 2471988,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978808.571, "dur": 4.054,
    "args": {
      "External id": 2471989,"Record function id": 0, "Ev Idx": 5684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978809.722, "dur": 2.491,
    "args": {
      "External id": 2471990,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978810.272, "dur": 1.534,
    "args": {
      "External id": 2471991,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978811.076, "dur": 0.622,
    "args": {
      "External id": 2471992,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978816.164, "dur": 36593.159,
    "args": {
      "External id": 2471993,"Record function id": 0, "Sequence number": 24740421, "Fwd thread id": 1, "Ev Idx": 5688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978817.404, "dur": 36582.991,
    "args": {
      "External id": 2471994,"Sequence number": 24740421, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5689
    }
  },
  {
    "ph": "f", "id": 227, "pid": 1336756, "tid": 1381189, "ts": 1514657978817.404,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.0)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978845.613, "dur": 38.173,
    "args": {
      "External id": 2471995,"Record function id": 0, "Ev Idx": 5690
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.0)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978891.431, "dur": 74.703,
    "args": {
      "External id": 2471996,"Record function id": 0, "Ev Idx": 5691
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.0)", "pid": 1336756, "tid": 1381189,
    "ts": 1514657978971.450, "dur": 36421.436,
    "args": {
      "External id": 2471997,"Record function id": 0, "Ev Idx": 5692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979119.176, "dur": 7.159,
    "args": {
      "External id": 2471998,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979136.582, "dur": 5.044,
    "args": {
      "External id": 2471999,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979160.335, "dur": 35450.859,
    "args": {
      "External id": 2472000,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979173.998, "dur": 35428.729,
    "args": {
      "External id": 2472001,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979229.761, "dur": 12.912,
    "args": {
      "External id": 2472002,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979250.465, "dur": 35312.644,
    "args": {
      "External id": 2472003,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979252.892, "dur": 35309.611,
    "args": {
      "External id": 2472004,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979256.474, "dur": 5.548,
    "args": {
      "External id": 2472005,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514657979263.679, "dur": 35295.564,
    "args": {
      "External id": 2472006,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514658014694.390, "dur": 7.972,
    "args": {
      "External id": 2472007,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514658014696.867, "dur": 5.096,
    "args": {
      "External id": 2472008,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514658014733.755, "dur": 377.216,
    "args": {
      "External id": 2472009,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514658014759.455, "dur": 346.198,
    "args": {
      "External id": 2472010,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5705, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514658014769.729, "dur": 330.021,
    "args": {
      "External id": 2472011,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015132.055, "dur": 2.424,
    "args": {
      "External id": 2472012,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5707, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015194.386, "dur": 6.233,
    "args": {
      "External id": 2472013,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015244.823, "dur": 1.271,
    "args": {
      "External id": 2472014,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015261.385, "dur": 2.609,
    "args": {
      "External id": 2472015,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015276.907, "dur": 1.112,
    "args": {
      "External id": 2472016,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015290.208, "dur": 0.986,
    "args": {
      "External id": 2472017,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015301.054, "dur": 0.881,
    "args": {
      "External id": 2472018,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015312.936, "dur": 2.059,
    "args": {
      "External id": 2472019,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015326.073, "dur": 0.896,
    "args": {
      "External id": 2472020,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015336.821, "dur": 0.812,
    "args": {
      "External id": 2472021,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015424.385, "dur": 294.835,
    "args": {
      "External id": 2472022,"Record function id": 0, "Sequence number": 24740420, "Fwd thread id": 1, "Ev Idx": 5717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015427.419, "dur": 283.811,
    "args": {
      "External id": 2472023,"Sequence number": 24740420, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5718
    }
  },
  {
    "ph": "f", "id": 228, "pid": 1336756, "tid": 1381189, "ts": 1514658015427.419,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_0", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015562.789, "dur": 46.911,
    "args": {
      "External id": 2472024,"kernel_hash": "cqirzqrltfdqnxhjos6hp6gemkm4gwku4foo6uqui7opbq4ng4m5", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/qi/cqirzqrltfdqnxhjos6hp6gemkm4gwku4foo6uqui7opbq4ng4m5.py", "kernel_backend": "triton", "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 5719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_1", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015626.533, "dur": 29.158,
    "args": {
      "External id": 2472025,"kernel_hash": "cjxnzfkkndlwdapjvgkkjvqxc72zinfqxyyoxiyd5swlefndj6n5", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/jx/cjxnzfkkndlwdapjvgkkjvqxc72zinfqxyyoxiyd5swlefndj6n5.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "float", "Scalar"], "Input Strides": [[4096, 1], [8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096], [16, 4096, 2048], [32000, 2048], []], "Ev Idx": 5720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_2", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015673.836, "dur": 22.415,
    "args": {
      "External id": 2472026,"kernel_hash": "cycx64w3vzu6rbiohboydiqd4wfbj7vazwzbvujyhnlqv6l7yuhz", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/yc/cycx64w3vzu6rbiohboydiqd4wfbj7vazwzbvujyhnlqv6l7yuhz.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 5721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015730.793, "dur": 15.733,
    "args": {
      "External id": 2472027,"Record function id": 0, "Ev Idx": 5722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015734.424, "dur": 11.318,
    "args": {
      "External id": 2472028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015738.706, "dur": 6.009,
    "args": {
      "External id": 2472029,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015740.283, "dur": 4.341,
    "args": {
      "External id": 2472030,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5725
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_post_backward_callback", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015767.772, "dur": 11339.292,
    "args": {
      "External id": 2472031,"Record function id": 0, "Ev Idx": 5726
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015788.374, "dur": 42.265,
    "args": {
      "External id": 2472032,"Record function id": 0, "Ev Idx": 5727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard", "pid": 1336756, "tid": 1381189,
    "ts": 1514658015836.826, "dur": 355.068,
    "args": {
      "External id": 2472033,"Record function id": 0, "Ev Idx": 5728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016200.384, "dur": 10596.320,
    "args": {
      "External id": 2472034,"Record function id": 0, "Ev Idx": 5729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016328.514, "dur": 7.215,
    "args": {
      "External id": 2472035,"Record function id": 0, "Concrete Inputs": ["[336611328]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016346.627, "dur": 5.409,
    "args": {
      "External id": 2472036,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016373.375, "dur": 9068.187,
    "args": {
      "External id": 2472037,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016392.194, "dur": 9035.876,
    "args": {
      "External id": 2472038,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016747.596, "dur": 20.713,
    "args": {
      "External id": 2472039,"Record function id": 0, "Concrete Inputs": ["[82421]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016862.787, "dur": 8511.047,
    "args": {
      "External id": 2472040,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], [], []], "Ev Idx": 5735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016866.219, "dur": 8506.530,
    "args": {
      "External id": 2472041,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], []], "Ev Idx": 5736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016871.209, "dur": 13.203,
    "args": {
      "External id": 2472042,"Record function id": 0, "Concrete Inputs": ["[82421]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1381189,
    "ts": 1514658016886.649, "dur": 8480.458,
    "args": {
      "External id": 2472043,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[82421], [82421], []], "Ev Idx": 5738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514658025601.849, "dur": 11.247,
    "args": {
      "External id": 2472044,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[336611328], [], [], [], [], []], "Ev Idx": 5739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1381189,
    "ts": 1514658025605.425, "dur": 7.100,
    "args": {
      "External id": 2472045,"Record function id": 0, "Concrete Inputs": ["[42076416]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336756, "tid": 1381189,
    "ts": 1514658025643.628, "dur": 493.221,
    "args": {
      "External id": 2472046,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[42076416], [336611328], [], [], [], []], "Ev Idx": 5741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514658025669.393, "dur": 461.930,
    "args": {
      "External id": 2472047,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 42076416, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[336611328], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5742, "In msg nelems": 336611328
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336756, "tid": 1381189,
    "ts": 1514658025685.114, "dur": 439.783,
    "args": {
      "External id": 2472048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[336611328]], "Ev Idx": 5743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026158.514, "dur": 2.868,
    "args": {
      "External id": 2472049,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5744, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026224.265, "dur": 7.247,
    "args": {
      "External id": 2472050,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026277.823, "dur": 1.206,
    "args": {
      "External id": 2472051,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026293.556, "dur": 1.455,
    "args": {
      "External id": 2472052,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8192256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026306.002, "dur": 1.967,
    "args": {
      "External id": 2472053,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8716544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026318.499, "dur": 1.148,
    "args": {
      "External id": 2472054,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9240832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026329.524, "dur": 1.340,
    "args": {
      "External id": 2472055,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9765120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026344.440, "dur": 1.224,
    "args": {
      "External id": 2472056,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026355.954, "dur": 2.471,
    "args": {
      "External id": 2472057,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "10289664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026369.380, "dur": 1.116,
    "args": {
      "External id": 2472058,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "11731456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026380.278, "dur": 1.127,
    "args": {
      "External id": 2472059,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "13173248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026391.491, "dur": 2.010,
    "args": {
      "External id": 2472060,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026403.803, "dur": 2.322,
    "args": {
      "External id": 2472061,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "14615296"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026416.056, "dur": 1.333,
    "args": {
      "External id": 2472062,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15139584"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026427.671, "dur": 1.413,
    "args": {
      "External id": 2472063,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15663872"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026440.153, "dur": 1.127,
    "args": {
      "External id": 2472064,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "16188160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026450.245, "dur": 2.836,
    "args": {
      "External id": 2472065,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026462.931, "dur": 1.636,
    "args": {
      "External id": 2472066,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "16712704"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026475.101, "dur": 0.982,
    "args": {
      "External id": 2472067,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "18154496"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026485.951, "dur": 1.294,
    "args": {
      "External id": 2472068,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "19596288"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026496.072, "dur": 1.980,
    "args": {
      "External id": 2472069,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026508.333, "dur": 1.155,
    "args": {
      "External id": 2472070,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21038336"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026519.112, "dur": 1.452,
    "args": {
      "External id": 2472071,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21562624"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026545.779, "dur": 1.981,
    "args": {
      "External id": 2472072,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22086912"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026562.139, "dur": 2.224,
    "args": {
      "External id": 2472073,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22611200"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026574.371, "dur": 1.265,
    "args": {
      "External id": 2472074,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026586.328, "dur": 1.178,
    "args": {
      "External id": 2472075,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "23135744"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026598.219, "dur": 1.450,
    "args": {
      "External id": 2472076,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "24577536"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026609.647, "dur": 2.185,
    "args": {
      "External id": 2472077,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "26019328"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026622.223, "dur": 1.040,
    "args": {
      "External id": 2472078,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026633.587, "dur": 1.379,
    "args": {
      "External id": 2472079,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27461376"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026644.854, "dur": 1.241,
    "args": {
      "External id": 2472080,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27985664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026655.733, "dur": 2.328,
    "args": {
      "External id": 2472081,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "28509952"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026667.569, "dur": 1.426,
    "args": {
      "External id": 2472082,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "29034240"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026678.501, "dur": 1.161,
    "args": {
      "External id": 2472083,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026689.064, "dur": 1.317,
    "args": {
      "External id": 2472084,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "29558784"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026699.961, "dur": 2.476,
    "args": {
      "External id": 2472085,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "31000576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026711.564, "dur": 1.090,
    "args": {
      "External id": 2472086,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "32442368"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026721.907, "dur": 1.278,
    "args": {
      "External id": 2472087,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1381189,
    "ts": 1514658026732.879, "dur": 1.293,
    "args": {
      "External id": 2472088,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "33884416"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5783
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "ProfilerStep#102911", "pid": 1336756, "tid": 1336756,
    "ts": 1514655937357.800, "dur": 2107836.979,
    "args": {
      "External id": 2461185,"Record function id": 0, "Ev Idx": 5784
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.zero_grad#AdamW.zero_grad", "pid": 1336756, "tid": 1336756,
    "ts": 1514655937391.773, "dur": 555.113,
    "args": {
      "External id": 2461186,"Record function id": 0, "Ev Idx": 5785
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "enumerate(DataLoader)#_StatefulMultiProcessingDataLoaderIter.__next__", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938020.848, "dur": 79.846,
    "args": {
      "External id": 2461187,"Record function id": 0, "Ev Idx": 5786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938797.022, "dur": 19.557,
    "args": {
      "External id": 2461188,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938809.336, "dur": 3.162,
    "args": {
      "External id": 2461189,"Record function id": 0, "Concrete Inputs": ["", "[16, 8192]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938818.532, "dur": 6.808,
    "args": {
      "External id": 2461190,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938823.407, "dur": 0.921,
    "args": {
      "External id": 2461191,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938847.555, "dur": 597.508,
    "args": {
      "External id": 2461192,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], [], []], "Ev Idx": 5791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938855.203, "dur": 589.225,
    "args": {
      "External id": 2461193,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938863.968, "dur": 10.186,
    "args": {
      "External id": 2461194,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938876.521, "dur": 565.993,
    "args": {
      "External id": 2461195,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938886.820, "dur": 0.289,
    "args": {
      "External id": 2461196,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938893.092, "dur": 8.062,
    "args": {
      "External id": 2461197,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[8192, 1], [4096, 1]], "Input Dims": [[16, 4096], [16, 4096]], "Ev Idx": 5796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938896.767, "dur": 4.260,
    "args": {
      "External id": 2461198,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], []], "Input Dims": [[16, 4096], [], []], "Ev Idx": 5797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938900.208, "dur": 0.568,
    "args": {
      "External id": 2461199,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938903.456, "dur": 287.457,
    "args": {
      "External id": 2461200,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938906.799, "dur": 283.700,
    "args": {
      "External id": 2461201,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938909.534, "dur": 15.589,
    "args": {
      "External id": 2461202,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 5801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938917.363, "dur": 7.297,
    "args": {
      "External id": 2461203,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655938928.645, "dur": 260.440,
    "args": {
      "External id": 2461204,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655939194.643, "dur": 244.293,
    "args": {
      "External id": 2461205,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655939460.874, "dur": 624.217,
    "args": {
      "External id": 2461206,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 5805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655939462.806, "dur": 621.549,
    "args": {
      "External id": 2461207,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], []], "Ev Idx": 5806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655939468.053, "dur": 8.701,
    "args": {
      "External id": 2461208,"Record function id": 0, "Concrete Inputs": ["[16, 8192]", "[8192, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655939478.126, "dur": 600.728,
    "args": {
      "External id": 2461209,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[8192, 1], [8192, 1], []], "Input Dims": [[16, 8192], [16, 8192], []], "Ev Idx": 5808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940120.282, "dur": 60.769,
    "args": {
      "External id": 2461210,"Record function id": 0, "Concrete Inputs": ["0", "4096", "", "", "", "False"], "Input type": ["Scalar", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940127.004, "dur": 6.513,
    "args": {
      "External id": 2461211,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940136.627, "dur": 44.104,
    "args": {
      "External id": 2461212,"Record function id": 0, "Concrete Inputs": ["0", "4096", "1", ""], "Input type": ["Scalar", "Scalar", "Scalar", "long int"], "Input Strides": [[], [], [], [1]], "Input Dims": [[], [], [], [0]], "Ev Idx": 5811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940143.285, "dur": 7.487,
    "args": {
      "External id": 2461213,"Record function id": 0, "Concrete Inputs": ["", "[4096]", ""], "Input type": ["long int", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 5812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::repeat", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940192.399, "dur": 74.111,
    "args": {
      "External id": 2461214,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4096], []], "Ev Idx": 5813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940198.011, "dur": 7.770,
    "args": {
      "External id": 2461215,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[4096], [], []], "Ev Idx": 5814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940203.509, "dur": 2.038,
    "args": {
      "External id": 2461216,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "[4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[4096], [], [], []], "Ev Idx": 5815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940209.353, "dur": 3.175,
    "args": {
      "External id": 2461217,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940215.153, "dur": 4.233,
    "args": {
      "External id": 2461218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[4096, 1]], "Input Dims": [[16, 4096]], "Ev Idx": 5817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940222.367, "dur": 7.494,
    "args": {
      "External id": 2461219,"Record function id": 0, "Concrete Inputs": ["", "0", "1", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940229.081, "dur": 0.602,
    "args": {
      "External id": 2461220,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 1]", "[4096, 1, 4096]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940230.870, "dur": 2.151,
    "args": {
      "External id": 2461221,"Record function id": 0, "Concrete Inputs": ["", "1", "4096", "4096"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940231.944, "dur": 0.984,
    "args": {
      "External id": 2461222,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[4096, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940235.141, "dur": 4.593,
    "args": {
      "External id": 2461223,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[4096, 1], [4096, 4096, 4096, 1]], "Input Dims": [[1, 4096], [16, 1, 1, 4096]], "Ev Idx": 5822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940237.021, "dur": 2.600,
    "args": {
      "External id": 2461224,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[4096, 1], [], []], "Input Dims": [[1, 4096], [], []], "Ev Idx": 5823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940238.779, "dur": 0.755,
    "args": {
      "External id": 2461225,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[0, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[1, 4096], [], [], []], "Ev Idx": 5824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940240.832, "dur": 24.954,
    "args": {
      "External id": 2461226,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 4096, 4096, 1], [0, 4096, 4096, 1], []], "Input Dims": [[16, 1, 1, 4096], [16, 1, 1, 4096], []], "Ev Idx": 5825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940274.430, "dur": 34.161,
    "args": {
      "External id": 2461227,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940278.724, "dur": 29.691,
    "args": {
      "External id": 2461228,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "", "", "", "False", ""], "Input type": ["long int", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940284.350, "dur": 3.584,
    "args": {
      "External id": 2461229,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "3", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940288.903, "dur": 19.082,
    "args": {
      "External id": 2461230,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5829
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_pre_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940416.735, "dur": 160.705,
    "args": {
      "External id": 2461231,"Record function id": 0, "Ev Idx": 5830
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::inputs_to_device", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940489.951, "dur": 76.564,
    "args": {
      "External id": 2461232,"Record function id": 0, "Ev Idx": 5831
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940584.417, "dur": 40.232,
    "args": {
      "External id": 2461233,"Record function id": 0, "Ev Idx": 5832
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940636.858, "dur": 11556.226,
    "args": {
      "External id": 2461234,"Record function id": 0, "Ev Idx": 5833
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940643.365, "dur": 1383.447,
    "args": {
      "External id": 2461235,"Record function id": 0, "Ev Idx": 5834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940749.959, "dur": 7.470,
    "args": {
      "External id": 2461236,"Record function id": 0, "Concrete Inputs": ["[42076416]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940775.580, "dur": 157.327,
    "args": {
      "External id": 2461237,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940780.942, "dur": 1.430,
    "args": {
      "External id": 2461238,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940787.093, "dur": 2.607,
    "args": {
      "External id": 2461239,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940790.471, "dur": 0.784,
    "args": {
      "External id": 2461240,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8192256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940791.816, "dur": 3.192,
    "args": {
      "External id": 2461241,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8716544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940801.377, "dur": 0.352,
    "args": {
      "External id": 2461242,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9240832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940802.590, "dur": 0.572,
    "args": {
      "External id": 2461243,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9765120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940806.540, "dur": 0.456,
    "args": {
      "External id": 2461244,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940809.975, "dur": 0.581,
    "args": {
      "External id": 2461245,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "10289664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940811.640, "dur": 0.584,
    "args": {
      "External id": 2461246,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11731456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940815.286, "dur": 3.025,
    "args": {
      "External id": 2461247,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "13173248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940819.617, "dur": 0.401,
    "args": {
      "External id": 2461248,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940820.999, "dur": 2.636,
    "args": {
      "External id": 2461249,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "14615296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940828.962, "dur": 0.357,
    "args": {
      "External id": 2461250,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15139584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940829.777, "dur": 0.702,
    "args": {
      "External id": 2461251,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15663872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940833.662, "dur": 0.500,
    "args": {
      "External id": 2461252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "16188160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940837.163, "dur": 0.584,
    "args": {
      "External id": 2461253,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940838.464, "dur": 0.547,
    "args": {
      "External id": 2461254,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "16712704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940842.228, "dur": 2.323,
    "args": {
      "External id": 2461255,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "18154496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940845.267, "dur": 0.712,
    "args": {
      "External id": 2461256,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "19596288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940846.524, "dur": 2.483,
    "args": {
      "External id": 2461257,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940854.830, "dur": 0.341,
    "args": {
      "External id": 2461258,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21038336"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940855.604, "dur": 0.561,
    "args": {
      "External id": 2461259,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21562624"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940859.165, "dur": 0.346,
    "args": {
      "External id": 2461260,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22086912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940862.328, "dur": 0.444,
    "args": {
      "External id": 2461261,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22611200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940863.229, "dur": 0.533,
    "args": {
      "External id": 2461262,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940866.982, "dur": 2.219,
    "args": {
      "External id": 2461263,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "23135744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940869.999, "dur": 0.627,
    "args": {
      "External id": 2461264,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24577536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940871.561, "dur": 2.042,
    "args": {
      "External id": 2461265,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "26019328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940878.950, "dur": 0.340,
    "args": {
      "External id": 2461266,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940879.947, "dur": 0.568,
    "args": {
      "External id": 2461267,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27461376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940883.566, "dur": 0.518,
    "args": {
      "External id": 2461268,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27985664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940886.692, "dur": 0.613,
    "args": {
      "External id": 2461269,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "28509952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940888.103, "dur": 0.390,
    "args": {
      "External id": 2461270,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "29034240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940891.626, "dur": 2.283,
    "args": {
      "External id": 2461271,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940894.400, "dur": 0.602,
    "args": {
      "External id": 2461272,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29558784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940895.721, "dur": 2.359,
    "args": {
      "External id": 2461273,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "31000576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940903.105, "dur": 0.405,
    "args": {
      "External id": 2461274,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "32442368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940904.154, "dur": 0.525,
    "args": {
      "External id": 2461275,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940907.235, "dur": 0.446,
    "args": {
      "External id": 2461276,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "33884416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655940955.701, "dur": 121.310,
    "args": {
      "External id": 2461277,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941151.840, "dur": 298.246,
    "args": {
      "External id": 2461278,"Record function id": 0, "Concrete Inputs": ["", "", "42076416", "8", "3", "15", ""], "Input type": ["TensorList", "", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 5877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941167.111, "dur": 6.042,
    "args": {
      "External id": 2461279,"Record function id": 0, "Concrete Inputs": ["[336611328]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941179.546, "dur": 17.022,
    "args": {
      "External id": 2461280,"Record function id": 0, "Concrete Inputs": ["", "0", "126229248", "42076416"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941186.077, "dur": 10.038,
    "args": {
      "External id": 2461281,"Record function id": 0, "Concrete Inputs": ["", "0", "126229248", "168305664", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[336611328], [], [], [], []], "Ev Idx": 5880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941192.248, "dur": 0.820,
    "args": {
      "External id": 2461282,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "[1]", "126229248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941205.047, "dur": 143.000,
    "args": {
      "External id": 2461283,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941206.980, "dur": 0.739,
    "args": {
      "External id": 2461284,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "126229248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941210.759, "dur": 0.795,
    "args": {
      "External id": 2461285,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "134421248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941212.593, "dur": 2.464,
    "args": {
      "External id": 2461286,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "134421504"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941215.914, "dur": 1.123,
    "args": {
      "External id": 2461287,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "134945792"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941220.688, "dur": 0.505,
    "args": {
      "External id": 2461288,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "135470080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941221.734, "dur": 0.142,
    "args": {
      "External id": 2461289,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "135994368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941223.238, "dur": 0.521,
    "args": {
      "External id": 2461290,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "136518656"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941227.004, "dur": 0.535,
    "args": {
      "External id": 2461291,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "136518912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941228.476, "dur": 0.728,
    "args": {
      "External id": 2461292,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "137960704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941232.402, "dur": 0.589,
    "args": {
      "External id": 2461293,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "139402496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941235.894, "dur": 2.428,
    "args": {
      "External id": 2461294,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "140844288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941239.009, "dur": 0.608,
    "args": {
      "External id": 2461295,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "140844544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941242.861, "dur": 0.347,
    "args": {
      "External id": 2461296,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "141368832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941246.276, "dur": 0.260,
    "args": {
      "External id": 2461297,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "141893120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941247.308, "dur": 0.796,
    "args": {
      "External id": 2461298,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "142417408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941250.986, "dur": 2.849,
    "args": {
      "External id": 2461299,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "142941696"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941254.522, "dur": 0.515,
    "args": {
      "External id": 2461300,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "142941952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941257.932, "dur": 0.508,
    "args": {
      "External id": 2461301,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "144383744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941261.414, "dur": 2.143,
    "args": {
      "External id": 2461302,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "145825536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941264.210, "dur": 0.549,
    "args": {
      "External id": 2461303,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "147267328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941268.125, "dur": 0.554,
    "args": {
      "External id": 2461304,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "147267584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941271.430, "dur": 0.360,
    "args": {
      "External id": 2461305,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "147791872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941272.974, "dur": 0.386,
    "args": {
      "External id": 2461306,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "148316160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941276.616, "dur": 2.747,
    "args": {
      "External id": 2461307,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "148840448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941280.226, "dur": 0.342,
    "args": {
      "External id": 2461308,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "149364736"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941283.588, "dur": 0.582,
    "args": {
      "External id": 2461309,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "149364992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941287.664, "dur": 2.679,
    "args": {
      "External id": 2461310,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "150806784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941290.939, "dur": 0.656,
    "args": {
      "External id": 2461311,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "152248576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941295.876, "dur": 0.576,
    "args": {
      "External id": 2461312,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "153690368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941299.003, "dur": 0.332,
    "args": {
      "External id": 2461313,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "153690624"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941300.273, "dur": 0.675,
    "args": {
      "External id": 2461314,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "154214912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941304.516, "dur": 2.743,
    "args": {
      "External id": 2461315,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "154739200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941308.393, "dur": 0.481,
    "args": {
      "External id": 2461316,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "155263488"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941311.728, "dur": 0.584,
    "args": {
      "External id": 2461317,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "155787776"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941315.084, "dur": 1.894,
    "args": {
      "External id": 2461318,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "155788032"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941317.655, "dur": 0.608,
    "args": {
      "External id": 2461319,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "157229824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941321.387, "dur": 0.406,
    "args": {
      "External id": 2461320,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "158671616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941324.797, "dur": 0.371,
    "args": {
      "External id": 2461321,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "160113408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941326.018, "dur": 0.376,
    "args": {
      "External id": 2461322,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "160113664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941371.816, "dur": 59.855,
    "args": {
      "External id": 2461323,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941507.354, "dur": 371.791,
    "args": {
      "External id": 2461324,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[336611328], [42076416], [], [], []], "Ev Idx": 5923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941564.368, "dur": 309.158,
    "args": {
      "External id": 2461325,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 336611328, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[42076416], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5924, "In msg nelems": 42076416
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941576.186, "dur": 291.747,
    "args": {
      "External id": 2461326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[42076416]], "Ev Idx": 5925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655941905.480, "dur": 2.524,
    "args": {
      "External id": 2461327,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5926, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942043.890, "dur": 9927.951,
    "args": {
      "External id": 2461328,"Record function id": 0, "Ev Idx": 5927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942242.698, "dur": 7.024,
    "args": {
      "External id": 2461329,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942253.230, "dur": 1.226,
    "args": {
      "External id": 2461330,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942256.154, "dur": 1.122,
    "args": {
      "External id": 2461331,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942258.714, "dur": 2.565,
    "args": {
      "External id": 2461332,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942264.982, "dur": 0.848,
    "args": {
      "External id": 2461333,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942267.489, "dur": 1.139,
    "args": {
      "External id": 2461334,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942270.239, "dur": 0.901,
    "args": {
      "External id": 2461335,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942274.787, "dur": 2.221,
    "args": {
      "External id": 2461336,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942280.646, "dur": 0.812,
    "args": {
      "External id": 2461337,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942283.067, "dur": 0.925,
    "args": {
      "External id": 2461338,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942285.492, "dur": 0.888,
    "args": {
      "External id": 2461339,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942287.710, "dur": 1.905,
    "args": {
      "External id": 2461340,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942293.472, "dur": 0.972,
    "args": {
      "External id": 2461341,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942295.677, "dur": 0.962,
    "args": {
      "External id": 2461342,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942298.120, "dur": 0.742,
    "args": {
      "External id": 2461343,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942302.937, "dur": 2.761,
    "args": {
      "External id": 2461344,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942309.824, "dur": 1.014,
    "args": {
      "External id": 2461345,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942312.147, "dur": 0.926,
    "args": {
      "External id": 2461346,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942314.371, "dur": 1.007,
    "args": {
      "External id": 2461347,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942316.993, "dur": 1.936,
    "args": {
      "External id": 2461348,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942322.279, "dur": 0.949,
    "args": {
      "External id": 2461349,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942324.612, "dur": 0.867,
    "args": {
      "External id": 2461350,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942326.687, "dur": 0.634,
    "args": {
      "External id": 2461351,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942331.114, "dur": 3.620,
    "args": {
      "External id": 2461352,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942338.743, "dur": 0.717,
    "args": {
      "External id": 2461353,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942340.824, "dur": 0.943,
    "args": {
      "External id": 2461354,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942343.118, "dur": 0.965,
    "args": {
      "External id": 2461355,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942345.486, "dur": 1.545,
    "args": {
      "External id": 2461356,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942350.718, "dur": 0.879,
    "args": {
      "External id": 2461357,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942352.786, "dur": 1.011,
    "args": {
      "External id": 2461358,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942355.192, "dur": 0.982,
    "args": {
      "External id": 2461359,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942359.726, "dur": 2.602,
    "args": {
      "External id": 2461360,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942365.540, "dur": 0.750,
    "args": {
      "External id": 2461361,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942367.556, "dur": 0.667,
    "args": {
      "External id": 2461362,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942369.577, "dur": 0.867,
    "args": {
      "External id": 2461363,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942371.680, "dur": 1.441,
    "args": {
      "External id": 2461364,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942376.888, "dur": 0.895,
    "args": {
      "External id": 2461365,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942392.409, "dur": 1.276,
    "args": {
      "External id": 2461366,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942394.910, "dur": 1.060,
    "args": {
      "External id": 2461367,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942399.486, "dur": 2.956,
    "args": {
      "External id": 2461368,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942431.278, "dur": 9485.467,
    "args": {
      "External id": 2461369,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942453.493, "dur": 9454.069,
    "args": {
      "External id": 2461370,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942479.426, "dur": 5.875,
    "args": {
      "External id": 2461371,"Record function id": 0, "Concrete Inputs": ["[4290]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942492.657, "dur": 9373.260,
    "args": {
      "External id": 2461372,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], [], []], "Ev Idx": 5971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942495.291, "dur": 9369.963,
    "args": {
      "External id": 2461373,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], []], "Ev Idx": 5972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942503.185, "dur": 5.192,
    "args": {
      "External id": 2461374,"Record function id": 0, "Concrete Inputs": ["[4290]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655942510.064, "dur": 9351.809,
    "args": {
      "External id": 2461375,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[4290], [4290], []], "Ev Idx": 5974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952271.375, "dur": 46.623,
    "args": {
      "External id": 2461376,"Record function id": 0, "Ev Idx": 5975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 0/0", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952319.283, "dur": 234.226,
    "args": {
      "External id": 2461377,"Record function id": 0, "Ev Idx": 5976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952361.023, "dur": 181.746,
    "args": {
      "External id": 2461378,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "long int"], "Input Strides": [[2048, 1], [4096, 1]], "Input Dims": [[32000, 2048], [16, 4096]], "Ev Idx": 5977
    }
  },
  {
    "ph": "s", "id": 228, "pid": 1336756, "tid": 1336756, "ts": 1514655952361.023,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952437.948, "dur": 51.279,
    "args": {
      "External id": 2461379,"kernel_hash": "ccn7soditjd7aij6esf6mo5c3yt5tfcdquxcuegqoe6voa34krv7", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/cn/ccn7soditjd7aij6esf6mo5c3yt5tfcdquxcuegqoe6voa34krv7.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[4096, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096], [32000, 2048], [16, 4096, 2048], []], "Ev Idx": 5978
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952616.531, "dur": 52.648,
    "args": {
      "External id": 2461380,"Record function id": 0, "Ev Idx": 5979
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.0)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952680.438, "dur": 8027.418,
    "args": {
      "External id": 2461381,"Record function id": 0, "Ev Idx": 5980
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952688.859, "dur": 804.750,
    "args": {
      "External id": 2461382,"Record function id": 0, "Ev Idx": 5981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952754.856, "dur": 11.545,
    "args": {
      "External id": 2461383,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952779.690, "dur": 45.337,
    "args": {
      "External id": 2461384,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952789.704, "dur": 2.377,
    "args": {
      "External id": 2461385,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952793.633, "dur": 0.577,
    "args": {
      "External id": 2461386,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952797.537, "dur": 2.841,
    "args": {
      "External id": 2461387,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952801.133, "dur": 0.352,
    "args": {
      "External id": 2461388,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952802.265, "dur": 0.330,
    "args": {
      "External id": 2461389,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952808.514, "dur": 0.145,
    "args": {
      "External id": 2461390,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952809.139, "dur": 0.375,
    "args": {
      "External id": 2461391,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952810.299, "dur": 1.400,
    "args": {
      "External id": 2461392,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952815.777, "dur": 0.314,
    "args": {
      "External id": 2461393,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952835.644, "dur": 36.031,
    "args": {
      "External id": 2461394,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952901.229, "dur": 181.525,
    "args": {
      "External id": 2461395,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952914.281, "dur": 3.912,
    "args": {
      "External id": 2461396,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952923.272, "dur": 9.057,
    "args": {
      "External id": 2461397,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952927.417, "dur": 4.498,
    "args": {
      "External id": 2461398,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952930.265, "dur": 0.533,
    "args": {
      "External id": 2461399,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952938.425, "dur": 35.604,
    "args": {
      "External id": 2461400,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952942.445, "dur": 0.355,
    "args": {
      "External id": 2461401,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952943.603, "dur": 0.477,
    "args": {
      "External id": 2461402,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952947.510, "dur": 0.310,
    "args": {
      "External id": 2461403,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952950.631, "dur": 0.622,
    "args": {
      "External id": 2461404,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952951.784, "dur": 3.555,
    "args": {
      "External id": 2461405,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952956.096, "dur": 0.379,
    "args": {
      "External id": 2461406,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952959.773, "dur": 0.656,
    "args": {
      "External id": 2461407,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952963.290, "dur": 0.573,
    "args": {
      "External id": 2461408,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655952967.111, "dur": 0.165,
    "args": {
      "External id": 2461409,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953042.571, "dur": 28.126,
    "args": {
      "External id": 2461410,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953136.548, "dur": 272.846,
    "args": {
      "External id": 2461411,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953167.760, "dur": 237.315,
    "args": {
      "External id": 2461412,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6011, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953177.100, "dur": 223.010,
    "args": {
      "External id": 2461413,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953431.728, "dur": 2.282,
    "args": {
      "External id": 2461414,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6013, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953511.855, "dur": 6968.473,
    "args": {
      "External id": 2461415,"Record function id": 0, "Ev Idx": 6014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953620.217, "dur": 6.519,
    "args": {
      "External id": 2461416,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953630.241, "dur": 0.671,
    "args": {
      "External id": 2461417,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953632.848, "dur": 0.936,
    "args": {
      "External id": 2461418,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953635.611, "dur": 2.636,
    "args": {
      "External id": 2461419,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953639.665, "dur": 1.277,
    "args": {
      "External id": 2461420,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953642.108, "dur": 0.861,
    "args": {
      "External id": 2461421,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953649.511, "dur": 0.849,
    "args": {
      "External id": 2461422,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953651.906, "dur": 2.615,
    "args": {
      "External id": 2461423,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953655.893, "dur": 1.067,
    "args": {
      "External id": 2461424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953658.171, "dur": 0.933,
    "args": {
      "External id": 2461425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953679.301, "dur": 6765.622,
    "args": {
      "External id": 2461426,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953696.877, "dur": 6741.405,
    "args": {
      "External id": 2461427,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953729.136, "dur": 13.906,
    "args": {
      "External id": 2461428,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953746.237, "dur": 6659.346,
    "args": {
      "External id": 2461429,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953748.988, "dur": 6656.102,
    "args": {
      "External id": 2461430,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953754.311, "dur": 6.130,
    "args": {
      "External id": 2461431,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655953762.237, "dur": 6639.722,
    "args": {
      "External id": 2461432,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514655960652.433, "dur": 31.063,
    "args": {
      "External id": 2461433,"Sequence number": 24740421, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6032
    }
  },
  {
    "ph": "s", "id": 227, "pid": 1336756, "tid": 1336756, "ts": 1514655960652.433,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514655960668.650, "dur": 10.262,
    "args": {
      "External id": 2461434,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655960674.551, "dur": 3.973,
    "args": {
      "External id": 2461435,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514655960749.348, "dur": 89.328,
    "args": {
      "External id": 2461436,"Record function id": 0, "Ev Idx": 6035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655960840.211, "dur": 1132.144,
    "args": {
      "External id": 2461437,"Record function id": 0, "Ev Idx": 6036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514655960879.987, "dur": 1079.667,
    "args": {
      "External id": 2461438,"Sequence number": 24740422, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6037
    }
  },
  {
    "ph": "s", "id": 226, "pid": 1336756, "tid": 1336756, "ts": 1514655960879.987,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514655960942.261, "dur": 93.047,
    "args": {
      "External id": 2461439,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961051.718, "dur": 107.731,
    "args": {
      "External id": 2461440,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961171.387, "dur": 37.989,
    "args": {
      "External id": 2461441,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961217.046, "dur": 31.190,
    "args": {
      "External id": 2461442,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961277.617, "dur": 29.376,
    "args": {
      "External id": 2461443,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961325.675, "dur": 16.693,
    "args": {
      "External id": 2461444,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961361.427, "dur": 130.307,
    "args": {
      "External id": 2461445,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961414.375, "dur": 9.847,
    "args": {
      "External id": 2461446,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961419.128, "dur": 4.409,
    "args": {
      "External id": 2461447,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961429.075, "dur": 3.991,
    "args": {
      "External id": 2461448,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961434.381, "dur": 0.912,
    "args": {
      "External id": 2461449,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961437.861, "dur": 3.012,
    "args": {
      "External id": 2461450,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961501.743, "dur": 79.522,
    "args": {
      "External id": 2461451,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961616.395, "dur": 30.367,
    "args": {
      "External id": 2461452,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961653.825, "dur": 43.778,
    "args": {
      "External id": 2461453,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961706.801, "dur": 36.516,
    "args": {
      "External id": 2461454,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961768.129, "dur": 25.708,
    "args": {
      "External id": 2461455,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961799.557, "dur": 34.480,
    "args": {
      "External id": 2461456,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655961860.305, "dur": 19.554,
    "args": {
      "External id": 2461457,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6056
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.0)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962092.596, "dur": 84.523,
    "args": {
      "External id": 2461458,"Record function id": 0, "Ev Idx": 6057
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962248.055, "dur": 44.155,
    "args": {
      "External id": 2461459,"Record function id": 0, "Ev Idx": 6058
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.1)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962300.875, "dur": 19103.755,
    "args": {
      "External id": 2461460,"Record function id": 0, "Ev Idx": 6059
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962311.742, "dur": 994.889,
    "args": {
      "External id": 2461461,"Record function id": 0, "Ev Idx": 6060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962388.379, "dur": 9.449,
    "args": {
      "External id": 2461462,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962412.720, "dur": 47.488,
    "args": {
      "External id": 2461463,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962418.101, "dur": 4.573,
    "args": {
      "External id": 2461464,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962424.117, "dur": 0.562,
    "args": {
      "External id": 2461465,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962428.744, "dur": 0.815,
    "args": {
      "External id": 2461466,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962432.618, "dur": 0.439,
    "args": {
      "External id": 2461467,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962434.004, "dur": 2.474,
    "args": {
      "External id": 2461468,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962439.683, "dur": 0.470,
    "args": {
      "External id": 2461469,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962443.765, "dur": 0.707,
    "args": {
      "External id": 2461470,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962448.117, "dur": 0.538,
    "args": {
      "External id": 2461471,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962449.678, "dur": 3.363,
    "args": {
      "External id": 2461472,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962471.711, "dur": 43.197,
    "args": {
      "External id": 2461473,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962583.270, "dur": 129.868,
    "args": {
      "External id": 2461474,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962594.573, "dur": 5.031,
    "args": {
      "External id": 2461475,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962604.987, "dur": 10.264,
    "args": {
      "External id": 2461476,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962609.424, "dur": 5.414,
    "args": {
      "External id": 2461477,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962612.572, "dur": 0.728,
    "args": {
      "External id": 2461478,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962621.976, "dur": 35.272,
    "args": {
      "External id": 2461479,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962626.663, "dur": 0.651,
    "args": {
      "External id": 2461480,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962628.428, "dur": 1.692,
    "args": {
      "External id": 2461481,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962631.021, "dur": 2.583,
    "args": {
      "External id": 2461482,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962636.584, "dur": 0.407,
    "args": {
      "External id": 2461483,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962637.889, "dur": 0.604,
    "args": {
      "External id": 2461484,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962644.362, "dur": 0.149,
    "args": {
      "External id": 2461485,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962645.423, "dur": 0.585,
    "args": {
      "External id": 2461486,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962646.442, "dur": 0.382,
    "args": {
      "External id": 2461487,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962652.508, "dur": 0.352,
    "args": {
      "External id": 2461488,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962675.273, "dur": 28.598,
    "args": {
      "External id": 2461489,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962767.432, "dur": 437.518,
    "args": {
      "External id": 2461490,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962802.330, "dur": 397.277,
    "args": {
      "External id": 2461491,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6090, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514655962811.683, "dur": 378.772,
    "args": {
      "External id": 2461492,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963232.736, "dur": 2.777,
    "args": {
      "External id": 2461493,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6092, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963327.749, "dur": 17891.971,
    "args": {
      "External id": 2461494,"Record function id": 0, "Ev Idx": 6093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963421.546, "dur": 6.298,
    "args": {
      "External id": 2461495,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963431.861, "dur": 0.919,
    "args": {
      "External id": 2461496,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963434.557, "dur": 0.642,
    "args": {
      "External id": 2461497,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963437.435, "dur": 1.015,
    "args": {
      "External id": 2461498,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963439.715, "dur": 1.073,
    "args": {
      "External id": 2461499,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963444.868, "dur": 0.981,
    "args": {
      "External id": 2461500,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963447.146, "dur": 0.482,
    "args": {
      "External id": 2461501,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963449.073, "dur": 3.194,
    "args": {
      "External id": 2461502,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963453.772, "dur": 1.033,
    "args": {
      "External id": 2461503,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963458.829, "dur": 0.746,
    "args": {
      "External id": 2461504,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963479.003, "dur": 17702.053,
    "args": {
      "External id": 2461505,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963494.366, "dur": 17679.585,
    "args": {
      "External id": 2461506,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963519.967, "dur": 46.216,
    "args": {
      "External id": 2461507,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963570.470, "dur": 17570.261,
    "args": {
      "External id": 2461508,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963573.096, "dur": 17567.092,
    "args": {
      "External id": 2461509,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963579.458, "dur": 5.861,
    "args": {
      "External id": 2461510,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655963587.281, "dur": 17549.790,
    "args": {
      "External id": 2461511,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981351.454, "dur": 29.595,
    "args": {
      "External id": 2461512,"Sequence number": 24740423, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6111
    }
  },
  {
    "ph": "s", "id": 225, "pid": 1336756, "tid": 1336756, "ts": 1514655981351.454,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981369.180, "dur": 7.447,
    "args": {
      "External id": 2461513,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981372.095, "dur": 4.319,
    "args": {
      "External id": 2461514,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981446.289, "dur": 79.104,
    "args": {
      "External id": 2461515,"Record function id": 0, "Ev Idx": 6114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981545.279, "dur": 1095.439,
    "args": {
      "External id": 2461516,"Record function id": 0, "Ev Idx": 6115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981587.921, "dur": 1039.255,
    "args": {
      "External id": 2461517,"Sequence number": 24740424, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6116
    }
  },
  {
    "ph": "s", "id": 224, "pid": 1336756, "tid": 1336756, "ts": 1514655981587.921,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981671.374, "dur": 44.277,
    "args": {
      "External id": 2461518,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981730.985, "dur": 102.990,
    "args": {
      "External id": 2461519,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981842.506, "dur": 38.067,
    "args": {
      "External id": 2461520,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981886.367, "dur": 30.488,
    "args": {
      "External id": 2461521,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655981947.639, "dur": 26.780,
    "args": {
      "External id": 2461522,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982025.001, "dur": 24.129,
    "args": {
      "External id": 2461523,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982065.629, "dur": 133.540,
    "args": {
      "External id": 2461524,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982116.295, "dur": 15.049,
    "args": {
      "External id": 2461525,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982123.307, "dur": 7.231,
    "args": {
      "External id": 2461526,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982133.803, "dur": 5.163,
    "args": {
      "External id": 2461527,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982140.393, "dur": 3.396,
    "args": {
      "External id": 2461528,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982146.085, "dur": 3.074,
    "args": {
      "External id": 2461529,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982210.475, "dur": 51.032,
    "args": {
      "External id": 2461530,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982292.858, "dur": 28.076,
    "args": {
      "External id": 2461531,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982330.071, "dur": 41.656,
    "args": {
      "External id": 2461532,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982377.634, "dur": 34.798,
    "args": {
      "External id": 2461533,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982438.036, "dur": 26.441,
    "args": {
      "External id": 2461534,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982469.523, "dur": 33.631,
    "args": {
      "External id": 2461535,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982522.444, "dur": 38.048,
    "args": {
      "External id": 2461536,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6135
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.1)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982705.457, "dur": 73.391,
    "args": {
      "External id": 2461537,"Record function id": 0, "Ev Idx": 6136
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982851.353, "dur": 43.729,
    "args": {
      "External id": 2461538,"Record function id": 0, "Ev Idx": 6137
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.2)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982903.950, "dur": 17947.033,
    "args": {
      "External id": 2461539,"Record function id": 0, "Ev Idx": 6138
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655982913.367, "dur": 919.373,
    "args": {
      "External id": 2461540,"Record function id": 0, "Ev Idx": 6139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983027.536, "dur": 9.496,
    "args": {
      "External id": 2461541,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983050.992, "dur": 41.560,
    "args": {
      "External id": 2461542,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983056.374, "dur": 2.380,
    "args": {
      "External id": 2461543,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983066.000, "dur": 0.628,
    "args": {
      "External id": 2461544,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983067.532, "dur": 0.310,
    "args": {
      "External id": 2461545,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983068.877, "dur": 0.516,
    "args": {
      "External id": 2461546,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983074.835, "dur": 0.405,
    "args": {
      "External id": 2461547,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983076.344, "dur": 0.622,
    "args": {
      "External id": 2461548,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983080.081, "dur": 2.826,
    "args": {
      "External id": 2461549,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983084.297, "dur": 0.448,
    "args": {
      "External id": 2461550,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983085.713, "dur": 0.659,
    "args": {
      "External id": 2461551,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983106.405, "dur": 45.038,
    "args": {
      "External id": 2461552,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983184.785, "dur": 119.997,
    "args": {
      "External id": 2461553,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983198.106, "dur": 3.162,
    "args": {
      "External id": 2461554,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983205.974, "dur": 10.166,
    "args": {
      "External id": 2461555,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983210.697, "dur": 5.016,
    "args": {
      "External id": 2461556,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983213.824, "dur": 0.731,
    "args": {
      "External id": 2461557,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983225.219, "dur": 36.791,
    "args": {
      "External id": 2461558,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983226.604, "dur": 2.919,
    "args": {
      "External id": 2461559,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983230.474, "dur": 0.556,
    "args": {
      "External id": 2461560,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983234.736, "dur": 0.294,
    "args": {
      "External id": 2461561,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983238.653, "dur": 1.456,
    "args": {
      "External id": 2461562,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983243.708, "dur": 0.398,
    "args": {
      "External id": 2461563,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983245.000, "dur": 0.152,
    "args": {
      "External id": 2461564,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983248.008, "dur": 0.377,
    "args": {
      "External id": 2461565,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983251.815, "dur": 0.643,
    "args": {
      "External id": 2461566,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983253.758, "dur": 2.636,
    "args": {
      "External id": 2461567,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983273.808, "dur": 23.410,
    "args": {
      "External id": 2461568,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983359.451, "dur": 379.448,
    "args": {
      "External id": 2461569,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983395.121, "dur": 338.604,
    "args": {
      "External id": 2461570,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6169, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983409.129, "dur": 318.175,
    "args": {
      "External id": 2461571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983765.756, "dur": 2.450,
    "args": {
      "External id": 2461572,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6171, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983851.527, "dur": 16819.450,
    "args": {
      "External id": 2461573,"Record function id": 0, "Ev Idx": 6172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983948.655, "dur": 6.411,
    "args": {
      "External id": 2461574,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983958.310, "dur": 1.137,
    "args": {
      "External id": 2461575,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983961.301, "dur": 2.295,
    "args": {
      "External id": 2461576,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983965.215, "dur": 1.039,
    "args": {
      "External id": 2461577,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983967.586, "dur": 0.929,
    "args": {
      "External id": 2461578,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983969.601, "dur": 0.878,
    "args": {
      "External id": 2461579,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983974.425, "dur": 1.052,
    "args": {
      "External id": 2461580,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655983977.157, "dur": 2.712,
    "args": {
      "External id": 2461581,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984015.381, "dur": 1.785,
    "args": {
      "External id": 2461582,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984020.759, "dur": 1.141,
    "args": {
      "External id": 2461583,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984045.565, "dur": 16586.647,
    "args": {
      "External id": 2461584,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984062.230, "dur": 16562.948,
    "args": {
      "External id": 2461585,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984088.349, "dur": 14.067,
    "args": {
      "External id": 2461586,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984105.347, "dur": 16485.857,
    "args": {
      "External id": 2461587,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984108.067, "dur": 16482.629,
    "args": {
      "External id": 2461588,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984116.510, "dur": 5.250,
    "args": {
      "External id": 2461589,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514655984123.675, "dur": 16464.195,
    "args": {
      "External id": 2461590,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656000800.620, "dur": 27.384,
    "args": {
      "External id": 2461591,"Sequence number": 24740425, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6190
    }
  },
  {
    "ph": "s", "id": 223, "pid": 1336756, "tid": 1336756, "ts": 1514656000800.620,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656000816.432, "dur": 6.960,
    "args": {
      "External id": 2461592,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656000819.389, "dur": 3.805,
    "args": {
      "External id": 2461593,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656000892.811, "dur": 76.142,
    "args": {
      "External id": 2461594,"Record function id": 0, "Ev Idx": 6193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656000970.469, "dur": 1196.046,
    "args": {
      "External id": 2461595,"Record function id": 0, "Ev Idx": 6194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001077.070, "dur": 1076.554,
    "args": {
      "External id": 2461596,"Sequence number": 24740426, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6195
    }
  },
  {
    "ph": "s", "id": 222, "pid": 1336756, "tid": 1336756, "ts": 1514656001077.070,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001144.698, "dur": 45.790,
    "args": {
      "External id": 2461597,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001203.042, "dur": 105.492,
    "args": {
      "External id": 2461598,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001318.152, "dur": 40.380,
    "args": {
      "External id": 2461599,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001368.048, "dur": 30.673,
    "args": {
      "External id": 2461600,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001424.270, "dur": 25.589,
    "args": {
      "External id": 2461601,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001468.642, "dur": 16.209,
    "args": {
      "External id": 2461602,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001502.245, "dur": 160.390,
    "args": {
      "External id": 2461603,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001580.465, "dur": 13.442,
    "args": {
      "External id": 2461604,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001586.824, "dur": 6.014,
    "args": {
      "External id": 2461605,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001596.733, "dur": 5.885,
    "args": {
      "External id": 2461606,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001603.953, "dur": 1.048,
    "args": {
      "External id": 2461607,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001607.484, "dur": 3.633,
    "args": {
      "External id": 2461608,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001674.041, "dur": 51.617,
    "args": {
      "External id": 2461609,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001757.073, "dur": 29.145,
    "args": {
      "External id": 2461610,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001794.968, "dur": 41.106,
    "args": {
      "External id": 2461611,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001844.006, "dur": 35.321,
    "args": {
      "External id": 2461612,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001900.661, "dur": 25.771,
    "args": {
      "External id": 2461613,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656001931.863, "dur": 34.980,
    "args": {
      "External id": 2461614,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002045.256, "dur": 26.321,
    "args": {
      "External id": 2461615,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6214
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.2)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002232.180, "dur": 76.049,
    "args": {
      "External id": 2461616,"Record function id": 0, "Ev Idx": 6215
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002382.114, "dur": 51.263,
    "args": {
      "External id": 2461617,"Record function id": 0, "Ev Idx": 6216
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.3)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002443.546, "dur": 18463.657,
    "args": {
      "External id": 2461618,"Record function id": 0, "Ev Idx": 6217
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002455.472, "dur": 1055.932,
    "args": {
      "External id": 2461619,"Record function id": 0, "Ev Idx": 6218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002561.426, "dur": 9.003,
    "args": {
      "External id": 2461620,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002584.519, "dur": 45.057,
    "args": {
      "External id": 2461621,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002589.871, "dur": 2.418,
    "args": {
      "External id": 2461622,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002599.291, "dur": 0.447,
    "args": {
      "External id": 2461623,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002600.911, "dur": 0.720,
    "args": {
      "External id": 2461624,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002604.923, "dur": 0.420,
    "args": {
      "External id": 2461625,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002608.446, "dur": 0.711,
    "args": {
      "External id": 2461626,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002610.065, "dur": 0.430,
    "args": {
      "External id": 2461627,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002614.135, "dur": 4.942,
    "args": {
      "External id": 2461628,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002620.011, "dur": 0.557,
    "args": {
      "External id": 2461629,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002621.518, "dur": 0.381,
    "args": {
      "External id": 2461630,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002644.088, "dur": 42.606,
    "args": {
      "External id": 2461631,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002725.562, "dur": 140.226,
    "args": {
      "External id": 2461632,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002739.428, "dur": 5.274,
    "args": {
      "External id": 2461633,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002749.980, "dur": 11.809,
    "args": {
      "External id": 2461634,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002754.264, "dur": 7.088,
    "args": {
      "External id": 2461635,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002759.260, "dur": 0.765,
    "args": {
      "External id": 2461636,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002767.889, "dur": 35.212,
    "args": {
      "External id": 2461637,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002769.801, "dur": 0.797,
    "args": {
      "External id": 2461638,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002773.946, "dur": 2.841,
    "args": {
      "External id": 2461639,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002777.713, "dur": 0.790,
    "args": {
      "External id": 2461640,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002779.178, "dur": 1.682,
    "args": {
      "External id": 2461641,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002786.477, "dur": 0.387,
    "args": {
      "External id": 2461642,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002787.349, "dur": 0.575,
    "args": {
      "External id": 2461643,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002789.091, "dur": 0.379,
    "args": {
      "External id": 2461644,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002794.756, "dur": 0.403,
    "args": {
      "External id": 2461645,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002796.506, "dur": 0.584,
    "args": {
      "External id": 2461646,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002830.853, "dur": 27.372,
    "args": {
      "External id": 2461647,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002919.258, "dur": 489.097,
    "args": {
      "External id": 2461648,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002953.708, "dur": 449.135,
    "args": {
      "External id": 2461649,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6248, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656002966.436, "dur": 429.747,
    "args": {
      "External id": 2461650,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003436.612, "dur": 2.788,
    "args": {
      "External id": 2461651,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6250, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003554.747, "dur": 17158.732,
    "args": {
      "External id": 2461652,"Record function id": 0, "Ev Idx": 6251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003655.003, "dur": 6.601,
    "args": {
      "External id": 2461653,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003665.135, "dur": 1.078,
    "args": {
      "External id": 2461654,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003667.935, "dur": 2.283,
    "args": {
      "External id": 2461655,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003671.910, "dur": 1.279,
    "args": {
      "External id": 2461656,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003674.740, "dur": 0.902,
    "args": {
      "External id": 2461657,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003677.085, "dur": 1.100,
    "args": {
      "External id": 2461658,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003681.991, "dur": 1.024,
    "args": {
      "External id": 2461659,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003684.640, "dur": 2.049,
    "args": {
      "External id": 2461660,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003688.185, "dur": 0.986,
    "args": {
      "External id": 2461661,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003692.752, "dur": 0.994,
    "args": {
      "External id": 2461662,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003713.606, "dur": 16958.021,
    "args": {
      "External id": 2461663,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003729.123, "dur": 16935.113,
    "args": {
      "External id": 2461664,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003752.581, "dur": 13.684,
    "args": {
      "External id": 2461665,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003769.203, "dur": 16861.150,
    "args": {
      "External id": 2461666,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003771.801, "dur": 16858.068,
    "args": {
      "External id": 2461667,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003779.733, "dur": 6.211,
    "args": {
      "External id": 2461668,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656003787.691, "dur": 16839.161,
    "args": {
      "External id": 2461669,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656020843.631, "dur": 39.461,
    "args": {
      "External id": 2461670,"Sequence number": 24740427, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6269
    }
  },
  {
    "ph": "s", "id": 221, "pid": 1336756, "tid": 1336756, "ts": 1514656020843.631,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656020861.930, "dur": 16.702,
    "args": {
      "External id": 2461671,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656020874.461, "dur": 3.930,
    "args": {
      "External id": 2461672,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656020949.262, "dur": 107.373,
    "args": {
      "External id": 2461673,"Record function id": 0, "Ev Idx": 6272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021062.329, "dur": 1086.258,
    "args": {
      "External id": 2461674,"Record function id": 0, "Ev Idx": 6273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021106.392, "dur": 1030.053,
    "args": {
      "External id": 2461675,"Sequence number": 24740428, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6274
    }
  },
  {
    "ph": "s", "id": 220, "pid": 1336756, "tid": 1336756, "ts": 1514656021106.392,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021174.692, "dur": 43.137,
    "args": {
      "External id": 2461676,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021230.212, "dur": 105.375,
    "args": {
      "External id": 2461677,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021348.238, "dur": 37.985,
    "args": {
      "External id": 2461678,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021392.118, "dur": 30.494,
    "args": {
      "External id": 2461679,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021450.106, "dur": 24.627,
    "args": {
      "External id": 2461680,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021491.686, "dur": 16.807,
    "args": {
      "External id": 2461681,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021544.793, "dur": 135.903,
    "args": {
      "External id": 2461682,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021598.667, "dur": 10.716,
    "args": {
      "External id": 2461683,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021603.198, "dur": 5.349,
    "args": {
      "External id": 2461684,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021611.967, "dur": 4.730,
    "args": {
      "External id": 2461685,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021618.143, "dur": 1.039,
    "args": {
      "External id": 2461686,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021623.895, "dur": 5.029,
    "args": {
      "External id": 2461687,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021692.797, "dur": 48.669,
    "args": {
      "External id": 2461688,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021774.069, "dur": 31.551,
    "args": {
      "External id": 2461689,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021813.849, "dur": 40.965,
    "args": {
      "External id": 2461690,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021862.076, "dur": 33.980,
    "args": {
      "External id": 2461691,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021916.450, "dur": 32.356,
    "args": {
      "External id": 2461692,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656021954.180, "dur": 67.941,
    "args": {
      "External id": 2461693,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022046.828, "dur": 21.145,
    "args": {
      "External id": 2461694,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6293
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.3)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022214.053, "dur": 74.348,
    "args": {
      "External id": 2461695,"Record function id": 0, "Ev Idx": 6294
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022357.065, "dur": 44.743,
    "args": {
      "External id": 2461696,"Record function id": 0, "Ev Idx": 6295
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.4)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022438.506, "dur": 17846.028,
    "args": {
      "External id": 2461697,"Record function id": 0, "Ev Idx": 6296
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022450.527, "dur": 919.171,
    "args": {
      "External id": 2461698,"Record function id": 0, "Ev Idx": 6297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022544.830, "dur": 8.587,
    "args": {
      "External id": 2461699,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022567.556, "dur": 41.788,
    "args": {
      "External id": 2461700,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022574.734, "dur": 2.146,
    "args": {
      "External id": 2461701,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022580.845, "dur": 0.557,
    "args": {
      "External id": 2461702,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022582.162, "dur": 0.447,
    "args": {
      "External id": 2461703,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022585.654, "dur": 2.809,
    "args": {
      "External id": 2461704,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022589.003, "dur": 0.169,
    "args": {
      "External id": 2461705,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022591.875, "dur": 0.144,
    "args": {
      "External id": 2461706,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022594.999, "dur": 2.357,
    "args": {
      "External id": 2461707,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022597.831, "dur": 0.174,
    "args": {
      "External id": 2461708,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022600.723, "dur": 0.338,
    "args": {
      "External id": 2461709,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022620.355, "dur": 44.820,
    "args": {
      "External id": 2461710,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022698.419, "dur": 116.229,
    "args": {
      "External id": 2461711,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022711.529, "dur": 3.514,
    "args": {
      "External id": 2461712,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022719.863, "dur": 13.098,
    "args": {
      "External id": 2461713,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022723.812, "dur": 8.699,
    "args": {
      "External id": 2461714,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022728.751, "dur": 2.657,
    "args": {
      "External id": 2461715,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022739.096, "dur": 31.634,
    "args": {
      "External id": 2461716,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022740.540, "dur": 0.442,
    "args": {
      "External id": 2461717,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022744.138, "dur": 0.530,
    "args": {
      "External id": 2461718,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022745.263, "dur": 0.328,
    "args": {
      "External id": 2461719,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022750.590, "dur": 2.314,
    "args": {
      "External id": 2461720,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022753.858, "dur": 0.339,
    "args": {
      "External id": 2461721,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022754.753, "dur": 2.374,
    "args": {
      "External id": 2461722,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022759.906, "dur": 0.425,
    "args": {
      "External id": 2461723,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022761.293, "dur": 0.302,
    "args": {
      "External id": 2461724,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022764.286, "dur": 0.333,
    "args": {
      "External id": 2461725,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022783.271, "dur": 23.675,
    "args": {
      "External id": 2461726,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022866.252, "dur": 405.183,
    "args": {
      "External id": 2461727,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022901.553, "dur": 364.822,
    "args": {
      "External id": 2461728,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6327, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656022912.557, "dur": 347.415,
    "args": {
      "External id": 2461729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023298.037, "dur": 2.532,
    "args": {
      "External id": 2461730,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6329, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023389.900, "dur": 16706.751,
    "args": {
      "External id": 2461731,"Record function id": 0, "Ev Idx": 6330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023486.350, "dur": 6.418,
    "args": {
      "External id": 2461732,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023496.163, "dur": 1.102,
    "args": {
      "External id": 2461733,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023498.990, "dur": 2.447,
    "args": {
      "External id": 2461734,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023502.930, "dur": 1.083,
    "args": {
      "External id": 2461735,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023505.686, "dur": 0.931,
    "args": {
      "External id": 2461736,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023508.029, "dur": 0.971,
    "args": {
      "External id": 2461737,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023513.043, "dur": 1.192,
    "args": {
      "External id": 2461738,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023517.229, "dur": 2.427,
    "args": {
      "External id": 2461739,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023520.964, "dur": 0.832,
    "args": {
      "External id": 2461740,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023523.010, "dur": 0.796,
    "args": {
      "External id": 2461741,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023561.521, "dur": 16494.356,
    "args": {
      "External id": 2461742,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023579.717, "dur": 16468.928,
    "args": {
      "External id": 2461743,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023601.310, "dur": 14.355,
    "args": {
      "External id": 2461744,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023618.764, "dur": 16392.961,
    "args": {
      "External id": 2461745,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023621.197, "dur": 16389.875,
    "args": {
      "External id": 2461746,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023626.990, "dur": 5.773,
    "args": {
      "External id": 2461747,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656023634.635, "dur": 16346.146,
    "args": {
      "External id": 2461748,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040229.005, "dur": 30.905,
    "args": {
      "External id": 2461749,"Sequence number": 24740429, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6348
    }
  },
  {
    "ph": "s", "id": 219, "pid": 1336756, "tid": 1336756, "ts": 1514656040229.005,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040244.828, "dur": 10.371,
    "args": {
      "External id": 2461750,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040250.661, "dur": 4.271,
    "args": {
      "External id": 2461751,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040325.458, "dur": 78.158,
    "args": {
      "External id": 2461752,"Record function id": 0, "Ev Idx": 6351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040405.054, "dur": 1114.139,
    "args": {
      "External id": 2461753,"Record function id": 0, "Ev Idx": 6352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040443.185, "dur": 1063.642,
    "args": {
      "External id": 2461754,"Sequence number": 24740430, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6353
    }
  },
  {
    "ph": "s", "id": 218, "pid": 1336756, "tid": 1336756, "ts": 1514656040443.185,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040508.373, "dur": 66.080,
    "args": {
      "External id": 2461755,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040590.385, "dur": 104.106,
    "args": {
      "External id": 2461756,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040703.595, "dur": 39.528,
    "args": {
      "External id": 2461757,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040752.280, "dur": 31.469,
    "args": {
      "External id": 2461758,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040810.297, "dur": 30.218,
    "args": {
      "External id": 2461759,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040863.375, "dur": 16.626,
    "args": {
      "External id": 2461760,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040898.963, "dur": 177.438,
    "args": {
      "External id": 2461761,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040949.712, "dur": 9.889,
    "args": {
      "External id": 2461762,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040954.007, "dur": 4.834,
    "args": {
      "External id": 2461763,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040961.998, "dur": 4.860,
    "args": {
      "External id": 2461764,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040969.727, "dur": 0.837,
    "args": {
      "External id": 2461765,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656040972.859, "dur": 5.420,
    "args": {
      "External id": 2461766,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041089.198, "dur": 52.826,
    "args": {
      "External id": 2461767,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041171.181, "dur": 29.790,
    "args": {
      "External id": 2461768,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041209.306, "dur": 41.927,
    "args": {
      "External id": 2461769,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041261.399, "dur": 35.158,
    "args": {
      "External id": 2461770,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041319.948, "dur": 30.767,
    "args": {
      "External id": 2461771,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041355.936, "dur": 33.531,
    "args": {
      "External id": 2461772,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041409.115, "dur": 19.096,
    "args": {
      "External id": 2461773,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6372
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.4)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041610.475, "dur": 76.829,
    "args": {
      "External id": 2461774,"Record function id": 0, "Ev Idx": 6373
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041760.763, "dur": 47.730,
    "args": {
      "External id": 2461775,"Record function id": 0, "Ev Idx": 6374
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.5)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041816.786, "dur": 18256.046,
    "args": {
      "External id": 2461776,"Record function id": 0, "Ev Idx": 6375
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041826.502, "dur": 918.906,
    "args": {
      "External id": 2461777,"Record function id": 0, "Ev Idx": 6376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041905.595, "dur": 8.293,
    "args": {
      "External id": 2461778,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041926.186, "dur": 40.103,
    "args": {
      "External id": 2461779,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041932.572, "dur": 2.296,
    "args": {
      "External id": 2461780,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041938.626, "dur": 0.266,
    "args": {
      "External id": 2461781,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041940.421, "dur": 0.225,
    "args": {
      "External id": 2461782,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041941.910, "dur": 0.362,
    "args": {
      "External id": 2461783,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041945.997, "dur": 0.439,
    "args": {
      "External id": 2461784,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041948.445, "dur": 0.338,
    "args": {
      "External id": 2461785,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041950.397, "dur": 3.923,
    "args": {
      "External id": 2461786,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041956.037, "dur": 0.472,
    "args": {
      "External id": 2461787,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041957.990, "dur": 0.492,
    "args": {
      "External id": 2461788,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656041977.185, "dur": 82.136,
    "args": {
      "External id": 2461789,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042096.844, "dur": 126.428,
    "args": {
      "External id": 2461790,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042111.272, "dur": 4.646,
    "args": {
      "External id": 2461791,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042120.896, "dur": 11.263,
    "args": {
      "External id": 2461792,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042125.298, "dur": 6.465,
    "args": {
      "External id": 2461793,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042129.557, "dur": 0.651,
    "args": {
      "External id": 2461794,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042139.195, "dur": 37.337,
    "args": {
      "External id": 2461795,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042141.828, "dur": 2.547,
    "args": {
      "External id": 2461796,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042146.526, "dur": 0.601,
    "args": {
      "External id": 2461797,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042148.876, "dur": 0.442,
    "args": {
      "External id": 2461798,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042154.749, "dur": 1.502,
    "args": {
      "External id": 2461799,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042157.773, "dur": 0.520,
    "args": {
      "External id": 2461800,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042160.038, "dur": 0.158,
    "args": {
      "External id": 2461801,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042163.903, "dur": 0.155,
    "args": {
      "External id": 2461802,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042165.873, "dur": 0.310,
    "args": {
      "External id": 2461803,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042167.968, "dur": 2.383,
    "args": {
      "External id": 2461804,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042189.774, "dur": 25.692,
    "args": {
      "External id": 2461805,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042278.377, "dur": 373.906,
    "args": {
      "External id": 2461806,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042314.019, "dur": 332.652,
    "args": {
      "External id": 2461807,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6406, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042325.220, "dur": 315.299,
    "args": {
      "External id": 2461808,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042676.837, "dur": 2.379,
    "args": {
      "External id": 2461809,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6408, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042764.616, "dur": 17090.931,
    "args": {
      "External id": 2461810,"Record function id": 0, "Ev Idx": 6409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042857.408, "dur": 6.312,
    "args": {
      "External id": 2461811,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042867.235, "dur": 1.404,
    "args": {
      "External id": 2461812,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042870.433, "dur": 2.736,
    "args": {
      "External id": 2461813,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042874.922, "dur": 1.193,
    "args": {
      "External id": 2461814,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042877.614, "dur": 1.432,
    "args": {
      "External id": 2461815,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042880.542, "dur": 1.213,
    "args": {
      "External id": 2461816,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042886.106, "dur": 1.122,
    "args": {
      "External id": 2461817,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042888.898, "dur": 3.613,
    "args": {
      "External id": 2461818,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042893.923, "dur": 1.290,
    "args": {
      "External id": 2461819,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042896.932, "dur": 0.665,
    "args": {
      "External id": 2461820,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042917.642, "dur": 16891.963,
    "args": {
      "External id": 2461821,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042939.710, "dur": 16862.613,
    "args": {
      "External id": 2461822,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042960.630, "dur": 13.138,
    "args": {
      "External id": 2461823,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042976.819, "dur": 16792.263,
    "args": {
      "External id": 2461824,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656042979.693, "dur": 16788.910,
    "args": {
      "External id": 2461825,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656043025.891, "dur": 5.801,
    "args": {
      "External id": 2461826,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656043033.532, "dur": 16732.031,
    "args": {
      "External id": 2461827,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060016.044, "dur": 30.428,
    "args": {
      "External id": 2461828,"Sequence number": 24740431, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6427
    }
  },
  {
    "ph": "s", "id": 217, "pid": 1336756, "tid": 1336756, "ts": 1514656060016.044,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060034.085, "dur": 7.874,
    "args": {
      "External id": 2461829,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060037.208, "dur": 4.396,
    "args": {
      "External id": 2461830,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060113.988, "dur": 79.553,
    "args": {
      "External id": 2461831,"Record function id": 0, "Ev Idx": 6430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060195.726, "dur": 1122.753,
    "args": {
      "External id": 2461832,"Record function id": 0, "Ev Idx": 6431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060242.060, "dur": 1063.829,
    "args": {
      "External id": 2461833,"Sequence number": 24740432, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6432
    }
  },
  {
    "ph": "s", "id": 216, "pid": 1336756, "tid": 1336756, "ts": 1514656060242.060,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060307.948, "dur": 43.068,
    "args": {
      "External id": 2461834,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060363.133, "dur": 108.897,
    "args": {
      "External id": 2461835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060481.616, "dur": 38.725,
    "args": {
      "External id": 2461836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060548.415, "dur": 36.365,
    "args": {
      "External id": 2461837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060613.666, "dur": 27.884,
    "args": {
      "External id": 2461838,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060660.179, "dur": 18.148,
    "args": {
      "External id": 2461839,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060696.877, "dur": 136.161,
    "args": {
      "External id": 2461840,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060746.419, "dur": 13.496,
    "args": {
      "External id": 2461841,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060751.248, "dur": 7.668,
    "args": {
      "External id": 2461842,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060762.558, "dur": 4.812,
    "args": {
      "External id": 2461843,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060768.596, "dur": 1.213,
    "args": {
      "External id": 2461844,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060778.724, "dur": 3.901,
    "args": {
      "External id": 2461845,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060844.222, "dur": 48.584,
    "args": {
      "External id": 2461846,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060926.501, "dur": 33.948,
    "args": {
      "External id": 2461847,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656060969.168, "dur": 81.552,
    "args": {
      "External id": 2461848,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061063.501, "dur": 37.394,
    "args": {
      "External id": 2461849,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061125.072, "dur": 29.165,
    "args": {
      "External id": 2461850,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061159.912, "dur": 32.877,
    "args": {
      "External id": 2461851,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061216.800, "dur": 20.762,
    "args": {
      "External id": 2461852,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6451
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.5)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061383.420, "dur": 75.871,
    "args": {
      "External id": 2461853,"Record function id": 0, "Ev Idx": 6452
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061550.943, "dur": 50.292,
    "args": {
      "External id": 2461854,"Record function id": 0, "Ev Idx": 6453
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.6)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061611.636, "dur": 17803.473,
    "args": {
      "External id": 2461855,"Record function id": 0, "Ev Idx": 6454
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061619.160, "dur": 929.112,
    "args": {
      "External id": 2461856,"Record function id": 0, "Ev Idx": 6455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061701.583, "dur": 7.727,
    "args": {
      "External id": 2461857,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061722.825, "dur": 42.092,
    "args": {
      "External id": 2461858,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061728.351, "dur": 2.348,
    "args": {
      "External id": 2461859,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061735.707, "dur": 0.410,
    "args": {
      "External id": 2461860,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061737.988, "dur": 0.618,
    "args": {
      "External id": 2461861,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061740.233, "dur": 0.357,
    "args": {
      "External id": 2461862,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061744.473, "dur": 0.287,
    "args": {
      "External id": 2461863,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061746.408, "dur": 0.450,
    "args": {
      "External id": 2461864,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061748.971, "dur": 4.243,
    "args": {
      "External id": 2461865,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061754.845, "dur": 0.534,
    "args": {
      "External id": 2461866,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061756.804, "dur": 0.419,
    "args": {
      "External id": 2461867,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061776.397, "dur": 40.677,
    "args": {
      "External id": 2461868,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061850.695, "dur": 117.085,
    "args": {
      "External id": 2461869,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061863.708, "dur": 3.644,
    "args": {
      "External id": 2461870,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061872.196, "dur": 10.525,
    "args": {
      "External id": 2461871,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061876.432, "dur": 5.848,
    "args": {
      "External id": 2461872,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061880.356, "dur": 0.598,
    "args": {
      "External id": 2461873,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061889.642, "dur": 35.993,
    "args": {
      "External id": 2461874,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061892.246, "dur": 2.652,
    "args": {
      "External id": 2461875,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061896.547, "dur": 0.399,
    "args": {
      "External id": 2461876,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061898.780, "dur": 0.325,
    "args": {
      "External id": 2461877,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061903.193, "dur": 2.147,
    "args": {
      "External id": 2461878,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061906.946, "dur": 0.356,
    "args": {
      "External id": 2461879,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061908.993, "dur": 0.367,
    "args": {
      "External id": 2461880,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061912.989, "dur": 0.147,
    "args": {
      "External id": 2461881,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061914.905, "dur": 0.397,
    "args": {
      "External id": 2461882,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061917.199, "dur": 2.630,
    "args": {
      "External id": 2461883,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656061936.167, "dur": 23.410,
    "args": {
      "External id": 2461884,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062060.211, "dur": 375.322,
    "args": {
      "External id": 2461885,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062096.581, "dur": 333.526,
    "args": {
      "External id": 2461886,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6485, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062108.719, "dur": 314.918,
    "args": {
      "External id": 2461887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062462.901, "dur": 2.413,
    "args": {
      "External id": 2461888,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6487, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062570.605, "dur": 16658.156,
    "args": {
      "External id": 2461889,"Record function id": 0, "Ev Idx": 6488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062669.760, "dur": 6.787,
    "args": {
      "External id": 2461890,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062680.742, "dur": 1.547,
    "args": {
      "External id": 2461891,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062684.381, "dur": 2.201,
    "args": {
      "External id": 2461892,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062688.796, "dur": 0.884,
    "args": {
      "External id": 2461893,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062691.574, "dur": 0.927,
    "args": {
      "External id": 2461894,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062693.987, "dur": 1.021,
    "args": {
      "External id": 2461895,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062699.199, "dur": 0.876,
    "args": {
      "External id": 2461896,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062701.854, "dur": 3.654,
    "args": {
      "External id": 2461897,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062707.338, "dur": 0.982,
    "args": {
      "External id": 2461898,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062709.944, "dur": 0.986,
    "args": {
      "External id": 2461899,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062731.278, "dur": 16451.814,
    "args": {
      "External id": 2461900,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062746.945, "dur": 16429.205,
    "args": {
      "External id": 2461901,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062769.824, "dur": 14.886,
    "args": {
      "External id": 2461902,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062787.836, "dur": 16354.852,
    "args": {
      "External id": 2461903,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062790.528, "dur": 16351.637,
    "args": {
      "External id": 2461904,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062796.696, "dur": 4.581,
    "args": {
      "External id": 2461905,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656062802.941, "dur": 16336.173,
    "args": {
      "External id": 2461906,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079363.773, "dur": 28.504,
    "args": {
      "External id": 2461907,"Sequence number": 24740433, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6506
    }
  },
  {
    "ph": "s", "id": 215, "pid": 1336756, "tid": 1336756, "ts": 1514656079363.773,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079380.844, "dur": 7.026,
    "args": {
      "External id": 2461908,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079383.746, "dur": 3.910,
    "args": {
      "External id": 2461909,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079455.170, "dur": 91.351,
    "args": {
      "External id": 2461910,"Record function id": 0, "Ev Idx": 6509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079549.474, "dur": 1074.345,
    "args": {
      "External id": 2461911,"Record function id": 0, "Ev Idx": 6510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079591.720, "dur": 1019.165,
    "args": {
      "External id": 2461912,"Sequence number": 24740434, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6511
    }
  },
  {
    "ph": "s", "id": 214, "pid": 1336756, "tid": 1336756, "ts": 1514656079591.720,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079658.550, "dur": 43.872,
    "args": {
      "External id": 2461913,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079714.747, "dur": 104.570,
    "args": {
      "External id": 2461914,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079828.187, "dur": 38.513,
    "args": {
      "External id": 2461915,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079876.445, "dur": 30.198,
    "args": {
      "External id": 2461916,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079930.889, "dur": 26.224,
    "args": {
      "External id": 2461917,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656079975.151, "dur": 50.845,
    "args": {
      "External id": 2461918,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080048.003, "dur": 133.593,
    "args": {
      "External id": 2461919,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080099.275, "dur": 12.068,
    "args": {
      "External id": 2461920,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080104.888, "dur": 5.736,
    "args": {
      "External id": 2461921,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080114.920, "dur": 7.008,
    "args": {
      "External id": 2461922,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080123.319, "dur": 1.247,
    "args": {
      "External id": 2461923,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080127.186, "dur": 3.914,
    "args": {
      "External id": 2461924,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080192.056, "dur": 50.071,
    "args": {
      "External id": 2461925,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080273.267, "dur": 31.509,
    "args": {
      "External id": 2461926,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080312.514, "dur": 39.480,
    "args": {
      "External id": 2461927,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080360.538, "dur": 33.655,
    "args": {
      "External id": 2461928,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080418.226, "dur": 27.508,
    "args": {
      "External id": 2461929,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080451.048, "dur": 34.485,
    "args": {
      "External id": 2461930,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080505.702, "dur": 18.219,
    "args": {
      "External id": 2461931,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6530
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.6)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080685.686, "dur": 77.049,
    "args": {
      "External id": 2461932,"Record function id": 0, "Ev Idx": 6531
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080837.137, "dur": 47.995,
    "args": {
      "External id": 2461933,"Record function id": 0, "Ev Idx": 6532
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.7)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080894.550, "dur": 17821.306,
    "args": {
      "External id": 2461934,"Record function id": 0, "Ev Idx": 6533
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656080903.404, "dur": 908.806,
    "args": {
      "External id": 2461935,"Record function id": 0, "Ev Idx": 6534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081017.453, "dur": 8.875,
    "args": {
      "External id": 2461936,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081041.502, "dur": 43.260,
    "args": {
      "External id": 2461937,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081047.167, "dur": 2.381,
    "args": {
      "External id": 2461938,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081054.524, "dur": 0.464,
    "args": {
      "External id": 2461939,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081056.634, "dur": 0.452,
    "args": {
      "External id": 2461940,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081058.760, "dur": 0.297,
    "args": {
      "External id": 2461941,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081063.053, "dur": 0.493,
    "args": {
      "External id": 2461942,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081065.622, "dur": 0.331,
    "args": {
      "External id": 2461943,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081067.971, "dur": 4.882,
    "args": {
      "External id": 2461944,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081074.926, "dur": 0.216,
    "args": {
      "External id": 2461945,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081076.752, "dur": 0.354,
    "args": {
      "External id": 2461946,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081097.060, "dur": 41.530,
    "args": {
      "External id": 2461947,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081173.231, "dur": 122.722,
    "args": {
      "External id": 2461948,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081187.105, "dur": 3.852,
    "args": {
      "External id": 2461949,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081195.856, "dur": 10.495,
    "args": {
      "External id": 2461950,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081200.069, "dur": 5.861,
    "args": {
      "External id": 2461951,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081203.965, "dur": 0.583,
    "args": {
      "External id": 2461952,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081212.979, "dur": 37.001,
    "args": {
      "External id": 2461953,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081215.619, "dur": 3.635,
    "args": {
      "External id": 2461954,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081220.631, "dur": 0.343,
    "args": {
      "External id": 2461955,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081222.645, "dur": 0.464,
    "args": {
      "External id": 2461956,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081227.181, "dur": 2.387,
    "args": {
      "External id": 2461957,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081230.992, "dur": 0.308,
    "args": {
      "External id": 2461958,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081232.932, "dur": 0.556,
    "args": {
      "External id": 2461959,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081237.004, "dur": 0.314,
    "args": {
      "External id": 2461960,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081239.192, "dur": 0.560,
    "args": {
      "External id": 2461961,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081241.372, "dur": 2.350,
    "args": {
      "External id": 2461962,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081262.803, "dur": 24.782,
    "args": {
      "External id": 2461963,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081349.715, "dur": 364.223,
    "args": {
      "External id": 2461964,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081383.900, "dur": 325.147,
    "args": {
      "External id": 2461965,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6564, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081395.204, "dur": 307.918,
    "args": {
      "External id": 2461966,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081741.805, "dur": 2.585,
    "args": {
      "External id": 2461967,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6566, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081831.684, "dur": 16678.171,
    "args": {
      "External id": 2461968,"Record function id": 0, "Ev Idx": 6567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081927.472, "dur": 5.978,
    "args": {
      "External id": 2461969,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081936.819, "dur": 1.255,
    "args": {
      "External id": 2461970,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081939.777, "dur": 2.629,
    "args": {
      "External id": 2461971,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081951.670, "dur": 0.830,
    "args": {
      "External id": 2461972,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081954.203, "dur": 0.938,
    "args": {
      "External id": 2461973,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081956.556, "dur": 1.187,
    "args": {
      "External id": 2461974,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081961.391, "dur": 0.822,
    "args": {
      "External id": 2461975,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081963.632, "dur": 2.396,
    "args": {
      "External id": 2461976,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081967.568, "dur": 0.796,
    "args": {
      "External id": 2461977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656081970.086, "dur": 1.006,
    "args": {
      "External id": 2461978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656082028.607, "dur": 16433.537,
    "args": {
      "External id": 2461979,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656082044.784, "dur": 16410.690,
    "args": {
      "External id": 2461980,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656082065.466, "dur": 14.069,
    "args": {
      "External id": 2461981,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656082082.774, "dur": 16339.873,
    "args": {
      "External id": 2461982,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656082085.615, "dur": 16336.372,
    "args": {
      "External id": 2461983,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656082091.350, "dur": 5.144,
    "args": {
      "External id": 2461984,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656082098.204, "dur": 16320.670,
    "args": {
      "External id": 2461985,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656098661.070, "dur": 30.667,
    "args": {
      "External id": 2461986,"Sequence number": 24740435, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6585
    }
  },
  {
    "ph": "s", "id": 213, "pid": 1336756, "tid": 1336756, "ts": 1514656098661.070,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656098679.494, "dur": 7.783,
    "args": {
      "External id": 2461987,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656098682.625, "dur": 4.285,
    "args": {
      "External id": 2461988,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656098757.901, "dur": 75.867,
    "args": {
      "External id": 2461989,"Record function id": 0, "Ev Idx": 6588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656098835.668, "dur": 1058.422,
    "args": {
      "External id": 2461990,"Record function id": 0, "Ev Idx": 6589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656098878.266, "dur": 1003.726,
    "args": {
      "External id": 2461991,"Sequence number": 24740436, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6590
    }
  },
  {
    "ph": "s", "id": 212, "pid": 1336756, "tid": 1336756, "ts": 1514656098878.266,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656098941.931, "dur": 71.214,
    "args": {
      "External id": 2461992,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099029.127, "dur": 94.982,
    "args": {
      "External id": 2461993,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099134.339, "dur": 35.893,
    "args": {
      "External id": 2461994,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099180.294, "dur": 30.768,
    "args": {
      "External id": 2461995,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099236.929, "dur": 28.057,
    "args": {
      "External id": 2461996,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099283.692, "dur": 16.913,
    "args": {
      "External id": 2461997,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099319.432, "dur": 128.926,
    "args": {
      "External id": 2461998,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099368.544, "dur": 10.325,
    "args": {
      "External id": 2461999,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099373.521, "dur": 4.607,
    "args": {
      "External id": 2462000,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099381.456, "dur": 6.446,
    "args": {
      "External id": 2462001,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099389.520, "dur": 1.182,
    "args": {
      "External id": 2462002,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099393.382, "dur": 3.610,
    "args": {
      "External id": 2462003,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099459.144, "dur": 43.204,
    "args": {
      "External id": 2462004,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099549.050, "dur": 29.398,
    "args": {
      "External id": 2462005,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099588.681, "dur": 44.527,
    "args": {
      "External id": 2462006,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099641.604, "dur": 33.803,
    "args": {
      "External id": 2462007,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099705.773, "dur": 29.183,
    "args": {
      "External id": 2462008,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099742.576, "dur": 33.797,
    "args": {
      "External id": 2462009,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099794.585, "dur": 18.415,
    "args": {
      "External id": 2462010,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6609
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.7)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656099956.847, "dur": 111.958,
    "args": {
      "External id": 2462011,"Record function id": 0, "Ev Idx": 6610
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100144.457, "dur": 45.825,
    "args": {
      "External id": 2462012,"Record function id": 0, "Ev Idx": 6611
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.8)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100199.735, "dur": 18157.133,
    "args": {
      "External id": 2462013,"Record function id": 0, "Ev Idx": 6612
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100207.454, "dur": 943.718,
    "args": {
      "External id": 2462014,"Record function id": 0, "Ev Idx": 6613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100288.917, "dur": 7.907,
    "args": {
      "External id": 2462015,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100309.760, "dur": 41.953,
    "args": {
      "External id": 2462016,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100315.355, "dur": 2.300,
    "args": {
      "External id": 2462017,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100322.379, "dur": 0.549,
    "args": {
      "External id": 2462018,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100324.779, "dur": 0.736,
    "args": {
      "External id": 2462019,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100327.216, "dur": 0.311,
    "args": {
      "External id": 2462020,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100331.741, "dur": 0.529,
    "args": {
      "External id": 2462021,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100333.561, "dur": 0.639,
    "args": {
      "External id": 2462022,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100335.408, "dur": 4.220,
    "args": {
      "External id": 2462023,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100341.293, "dur": 0.603,
    "args": {
      "External id": 2462024,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100343.371, "dur": 0.501,
    "args": {
      "External id": 2462025,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100364.102, "dur": 41.740,
    "args": {
      "External id": 2462026,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100438.725, "dur": 133.995,
    "args": {
      "External id": 2462027,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100452.648, "dur": 3.676,
    "args": {
      "External id": 2462028,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100461.387, "dur": 11.125,
    "args": {
      "External id": 2462029,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100465.685, "dur": 6.400,
    "args": {
      "External id": 2462030,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100469.818, "dur": 0.677,
    "args": {
      "External id": 2462031,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100478.961, "dur": 34.207,
    "args": {
      "External id": 2462032,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100481.349, "dur": 2.612,
    "args": {
      "External id": 2462033,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100485.237, "dur": 0.523,
    "args": {
      "External id": 2462034,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100487.725, "dur": 0.504,
    "args": {
      "External id": 2462035,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100492.909, "dur": 2.378,
    "args": {
      "External id": 2462036,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100496.540, "dur": 0.289,
    "args": {
      "External id": 2462037,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100498.033, "dur": 0.374,
    "args": {
      "External id": 2462038,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100501.778, "dur": 0.589,
    "args": {
      "External id": 2462039,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100503.423, "dur": 0.297,
    "args": {
      "External id": 2462040,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100505.141, "dur": 2.472,
    "args": {
      "External id": 2462041,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100522.267, "dur": 40.187,
    "args": {
      "External id": 2462042,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100628.774, "dur": 415.685,
    "args": {
      "External id": 2462043,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100663.417, "dur": 375.519,
    "args": {
      "External id": 2462044,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6643, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656100674.949, "dur": 356.805,
    "args": {
      "External id": 2462045,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101073.819, "dur": 3.238,
    "args": {
      "External id": 2462046,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6645, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101171.167, "dur": 16999.833,
    "args": {
      "External id": 2462047,"Record function id": 0, "Ev Idx": 6646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101272.528, "dur": 6.505,
    "args": {
      "External id": 2462048,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101282.848, "dur": 1.430,
    "args": {
      "External id": 2462049,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101286.164, "dur": 2.167,
    "args": {
      "External id": 2462050,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101289.903, "dur": 1.191,
    "args": {
      "External id": 2462051,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101292.524, "dur": 1.189,
    "args": {
      "External id": 2462052,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101295.113, "dur": 0.740,
    "args": {
      "External id": 2462053,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101300.100, "dur": 1.144,
    "args": {
      "External id": 2462054,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101302.597, "dur": 2.291,
    "args": {
      "External id": 2462055,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101306.106, "dur": 1.013,
    "args": {
      "External id": 2462056,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101308.390, "dur": 0.940,
    "args": {
      "External id": 2462057,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101329.959, "dur": 16795.535,
    "args": {
      "External id": 2462058,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101345.365, "dur": 16773.147,
    "args": {
      "External id": 2462059,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101368.036, "dur": 14.167,
    "args": {
      "External id": 2462060,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101385.266, "dur": 16699.528,
    "args": {
      "External id": 2462061,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101387.572, "dur": 16696.677,
    "args": {
      "External id": 2462062,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101394.069, "dur": 4.909,
    "args": {
      "External id": 2462063,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656101400.527, "dur": 16680.650,
    "args": {
      "External id": 2462064,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118305.991, "dur": 27.449,
    "args": {
      "External id": 2462065,"Sequence number": 24740437, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6664
    }
  },
  {
    "ph": "s", "id": 211, "pid": 1336756, "tid": 1336756, "ts": 1514656118305.991,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118321.834, "dur": 7.149,
    "args": {
      "External id": 2462066,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118325.017, "dur": 3.755,
    "args": {
      "External id": 2462067,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118396.830, "dur": 77.162,
    "args": {
      "External id": 2462068,"Record function id": 0, "Ev Idx": 6667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118475.554, "dur": 1098.648,
    "args": {
      "External id": 2462069,"Record function id": 0, "Ev Idx": 6668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118514.465, "dur": 1045.622,
    "args": {
      "External id": 2462070,"Sequence number": 24740438, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6669
    }
  },
  {
    "ph": "s", "id": 210, "pid": 1336756, "tid": 1336756, "ts": 1514656118514.465,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118602.860, "dur": 44.269,
    "args": {
      "External id": 2462071,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118663.253, "dur": 105.411,
    "args": {
      "External id": 2462072,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118777.614, "dur": 38.066,
    "args": {
      "External id": 2462073,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118821.626, "dur": 30.445,
    "args": {
      "External id": 2462074,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118882.871, "dur": 28.312,
    "args": {
      "External id": 2462075,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118927.748, "dur": 18.603,
    "args": {
      "External id": 2462076,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656118961.951, "dur": 173.105,
    "args": {
      "External id": 2462077,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119051.425, "dur": 10.968,
    "args": {
      "External id": 2462078,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119056.016, "dur": 5.539,
    "args": {
      "External id": 2462079,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119065.419, "dur": 5.402,
    "args": {
      "External id": 2462080,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119072.187, "dur": 3.331,
    "args": {
      "External id": 2462081,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119077.729, "dur": 5.088,
    "args": {
      "External id": 2462082,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119147.142, "dur": 51.014,
    "args": {
      "External id": 2462083,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119228.585, "dur": 28.893,
    "args": {
      "External id": 2462084,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119266.425, "dur": 40.261,
    "args": {
      "External id": 2462085,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119315.020, "dur": 34.159,
    "args": {
      "External id": 2462086,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119373.860, "dur": 27.788,
    "args": {
      "External id": 2462087,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119407.519, "dur": 33.477,
    "args": {
      "External id": 2462088,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119457.573, "dur": 21.535,
    "args": {
      "External id": 2462089,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6688
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.8)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119639.454, "dur": 75.514,
    "args": {
      "External id": 2462090,"Record function id": 0, "Ev Idx": 6689
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119788.780, "dur": 45.511,
    "args": {
      "External id": 2462091,"Record function id": 0, "Ev Idx": 6690
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.9)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119843.817, "dur": 17953.969,
    "args": {
      "External id": 2462092,"Record function id": 0, "Ev Idx": 6691
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119852.552, "dur": 984.395,
    "args": {
      "External id": 2462093,"Record function id": 0, "Ev Idx": 6692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119932.640, "dur": 8.045,
    "args": {
      "External id": 2462094,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119954.482, "dur": 73.564,
    "args": {
      "External id": 2462095,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119959.602, "dur": 2.159,
    "args": {
      "External id": 2462096,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119966.209, "dur": 0.391,
    "args": {
      "External id": 2462097,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119967.913, "dur": 0.429,
    "args": {
      "External id": 2462098,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119969.225, "dur": 0.384,
    "args": {
      "External id": 2462099,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119973.316, "dur": 0.503,
    "args": {
      "External id": 2462100,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119975.074, "dur": 0.447,
    "args": {
      "External id": 2462101,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656119976.692, "dur": 3.408,
    "args": {
      "External id": 2462102,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120016.789, "dur": 0.637,
    "args": {
      "External id": 2462103,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120019.767, "dur": 0.477,
    "args": {
      "External id": 2462104,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120040.748, "dur": 42.805,
    "args": {
      "External id": 2462105,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120118.557, "dur": 127.446,
    "args": {
      "External id": 2462106,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120131.472, "dur": 4.571,
    "args": {
      "External id": 2462107,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120141.229, "dur": 9.390,
    "args": {
      "External id": 2462108,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120145.459, "dur": 4.723,
    "args": {
      "External id": 2462109,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120148.587, "dur": 0.442,
    "args": {
      "External id": 2462110,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120157.279, "dur": 35.616,
    "args": {
      "External id": 2462111,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120159.019, "dur": 2.857,
    "args": {
      "External id": 2462112,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120163.231, "dur": 0.396,
    "args": {
      "External id": 2462113,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120164.864, "dur": 0.445,
    "args": {
      "External id": 2462114,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120168.931, "dur": 1.862,
    "args": {
      "External id": 2462115,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120175.732, "dur": 0.422,
    "args": {
      "External id": 2462116,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120178.322, "dur": 0.212,
    "args": {
      "External id": 2462117,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120181.792, "dur": 0.584,
    "args": {
      "External id": 2462118,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120183.210, "dur": 0.513,
    "args": {
      "External id": 2462119,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120185.108, "dur": 2.237,
    "args": {
      "External id": 2462120,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120213.496, "dur": 24.438,
    "args": {
      "External id": 2462121,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120300.563, "dur": 434.723,
    "args": {
      "External id": 2462122,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120334.060, "dur": 395.798,
    "args": {
      "External id": 2462123,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6722, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120349.309, "dur": 372.244,
    "args": {
      "External id": 2462124,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120763.417, "dur": 2.120,
    "args": {
      "External id": 2462125,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6724, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120856.885, "dur": 16756.084,
    "args": {
      "External id": 2462126,"Record function id": 0, "Ev Idx": 6725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120954.204, "dur": 6.457,
    "args": {
      "External id": 2462127,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120964.269, "dur": 1.240,
    "args": {
      "External id": 2462128,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120967.284, "dur": 2.139,
    "args": {
      "External id": 2462129,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120971.087, "dur": 1.489,
    "args": {
      "External id": 2462130,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120973.994, "dur": 0.995,
    "args": {
      "External id": 2462131,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656120976.431, "dur": 0.769,
    "args": {
      "External id": 2462132,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121014.054, "dur": 1.544,
    "args": {
      "External id": 2462133,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121018.725, "dur": 2.902,
    "args": {
      "External id": 2462134,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121023.064, "dur": 0.737,
    "args": {
      "External id": 2462135,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121025.309, "dur": 0.789,
    "args": {
      "External id": 2462136,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121047.536, "dur": 16516.176,
    "args": {
      "External id": 2462137,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121063.689, "dur": 16492.683,
    "args": {
      "External id": 2462138,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121085.689, "dur": 14.377,
    "args": {
      "External id": 2462139,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121103.092, "dur": 16407.292,
    "args": {
      "External id": 2462140,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121105.658, "dur": 16404.235,
    "args": {
      "External id": 2462141,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121111.302, "dur": 6.128,
    "args": {
      "External id": 2462142,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656121119.286, "dur": 16387.588,
    "args": {
      "External id": 2462143,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656137746.208, "dur": 28.444,
    "args": {
      "External id": 2462144,"Sequence number": 24740439, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6743
    }
  },
  {
    "ph": "s", "id": 209, "pid": 1336756, "tid": 1336756, "ts": 1514656137746.208,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656137762.696, "dur": 7.377,
    "args": {
      "External id": 2462145,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656137766.146, "dur": 3.695,
    "args": {
      "External id": 2462146,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656137840.644, "dur": 76.994,
    "args": {
      "External id": 2462147,"Record function id": 0, "Ev Idx": 6746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656137919.347, "dur": 1098.024,
    "args": {
      "External id": 2462148,"Record function id": 0, "Ev Idx": 6747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656137958.115, "dur": 1012.318,
    "args": {
      "External id": 2462149,"Sequence number": 24740440, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6748
    }
  },
  {
    "ph": "s", "id": 208, "pid": 1336756, "tid": 1336756, "ts": 1514656137958.115,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138053.113, "dur": 43.207,
    "args": {
      "External id": 2462150,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138109.633, "dur": 102.696,
    "args": {
      "External id": 2462151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138222.369, "dur": 39.146,
    "args": {
      "External id": 2462152,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138269.240, "dur": 30.443,
    "args": {
      "External id": 2462153,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138325.331, "dur": 25.323,
    "args": {
      "External id": 2462154,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138370.623, "dur": 16.440,
    "args": {
      "External id": 2462155,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138406.395, "dur": 147.783,
    "args": {
      "External id": 2462156,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138456.875, "dur": 10.545,
    "args": {
      "External id": 2462157,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138461.611, "dur": 4.913,
    "args": {
      "External id": 2462158,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138470.439, "dur": 5.765,
    "args": {
      "External id": 2462159,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138477.981, "dur": 0.905,
    "args": {
      "External id": 2462160,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138481.308, "dur": 5.289,
    "args": {
      "External id": 2462161,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138567.313, "dur": 51.642,
    "args": {
      "External id": 2462162,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138647.615, "dur": 30.498,
    "args": {
      "External id": 2462163,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138689.182, "dur": 41.551,
    "args": {
      "External id": 2462164,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138738.738, "dur": 34.917,
    "args": {
      "External id": 2462165,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138795.935, "dur": 29.440,
    "args": {
      "External id": 2462166,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138831.464, "dur": 33.440,
    "args": {
      "External id": 2462167,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656138885.417, "dur": 17.291,
    "args": {
      "External id": 2462168,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6767
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.9)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139085.068, "dur": 75.970,
    "args": {
      "External id": 2462169,"Record function id": 0, "Ev Idx": 6768
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139236.575, "dur": 47.297,
    "args": {
      "External id": 2462170,"Record function id": 0, "Ev Idx": 6769
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.10)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139293.425, "dur": 17816.705,
    "args": {
      "External id": 2462171,"Record function id": 0, "Ev Idx": 6770
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139302.439, "dur": 920.749,
    "args": {
      "External id": 2462172,"Record function id": 0, "Ev Idx": 6771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139385.886, "dur": 8.303,
    "args": {
      "External id": 2462173,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139406.610, "dur": 42.147,
    "args": {
      "External id": 2462174,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139414.077, "dur": 2.430,
    "args": {
      "External id": 2462175,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139421.589, "dur": 0.717,
    "args": {
      "External id": 2462176,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139423.966, "dur": 0.365,
    "args": {
      "External id": 2462177,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139425.817, "dur": 2.605,
    "args": {
      "External id": 2462178,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139430.190, "dur": 0.498,
    "args": {
      "External id": 2462179,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139432.210, "dur": 0.478,
    "args": {
      "External id": 2462180,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139436.921, "dur": 1.705,
    "args": {
      "External id": 2462181,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139440.223, "dur": 0.194,
    "args": {
      "External id": 2462182,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139441.983, "dur": 0.319,
    "args": {
      "External id": 2462183,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139461.313, "dur": 40.321,
    "args": {
      "External id": 2462184,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139551.005, "dur": 126.223,
    "args": {
      "External id": 2462185,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139565.323, "dur": 5.167,
    "args": {
      "External id": 2462186,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139575.790, "dur": 12.840,
    "args": {
      "External id": 2462187,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139580.123, "dur": 8.064,
    "args": {
      "External id": 2462188,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139584.013, "dur": 2.655,
    "args": {
      "External id": 2462189,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139596.255, "dur": 33.139,
    "args": {
      "External id": 2462190,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139598.812, "dur": 0.484,
    "args": {
      "External id": 2462191,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139601.141, "dur": 0.441,
    "args": {
      "External id": 2462192,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139603.348, "dur": 0.558,
    "args": {
      "External id": 2462193,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139607.815, "dur": 1.793,
    "args": {
      "External id": 2462194,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139611.148, "dur": 0.315,
    "args": {
      "External id": 2462195,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139613.316, "dur": 2.616,
    "args": {
      "External id": 2462196,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139617.526, "dur": 0.165,
    "args": {
      "External id": 2462197,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139618.987, "dur": 0.298,
    "args": {
      "External id": 2462198,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139623.461, "dur": 0.377,
    "args": {
      "External id": 2462199,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139641.784, "dur": 26.856,
    "args": {
      "External id": 2462200,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139732.129, "dur": 389.915,
    "args": {
      "External id": 2462201,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139764.242, "dur": 352.227,
    "args": {
      "External id": 2462202,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6801, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656139776.700, "dur": 333.537,
    "args": {
      "External id": 2462203,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140150.471, "dur": 2.482,
    "args": {
      "External id": 2462204,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6803, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140243.870, "dur": 16648.460,
    "args": {
      "External id": 2462205,"Record function id": 0, "Ev Idx": 6804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140339.891, "dur": 6.243,
    "args": {
      "External id": 2462206,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140349.583, "dur": 1.035,
    "args": {
      "External id": 2462207,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140352.677, "dur": 2.145,
    "args": {
      "External id": 2462208,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140382.215, "dur": 0.956,
    "args": {
      "External id": 2462209,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140384.563, "dur": 1.052,
    "args": {
      "External id": 2462210,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140386.789, "dur": 0.941,
    "args": {
      "External id": 2462211,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140391.610, "dur": 0.978,
    "args": {
      "External id": 2462212,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140394.187, "dur": 2.061,
    "args": {
      "External id": 2462213,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140397.796, "dur": 0.764,
    "args": {
      "External id": 2462214,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140399.994, "dur": 0.907,
    "args": {
      "External id": 2462215,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140423.389, "dur": 16430.555,
    "args": {
      "External id": 2462216,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140439.499, "dur": 16407.417,
    "args": {
      "External id": 2462217,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140453.393, "dur": 13.525,
    "args": {
      "External id": 2462218,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140469.797, "dur": 16343.955,
    "args": {
      "External id": 2462219,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140472.015, "dur": 16340.954,
    "args": {
      "External id": 2462220,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140478.635, "dur": 6.016,
    "args": {
      "External id": 2462221,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656140486.510, "dur": 16323.750,
    "args": {
      "External id": 2462222,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157048.444, "dur": 36.063,
    "args": {
      "External id": 2462223,"Sequence number": 24740441, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6822
    }
  },
  {
    "ph": "s", "id": 207, "pid": 1336756, "tid": 1336756, "ts": 1514656157048.444,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157071.512, "dur": 8.107,
    "args": {
      "External id": 2462224,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157074.795, "dur": 4.378,
    "args": {
      "External id": 2462225,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157150.153, "dur": 77.686,
    "args": {
      "External id": 2462226,"Record function id": 0, "Ev Idx": 6825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157229.397, "dur": 1081.073,
    "args": {
      "External id": 2462227,"Record function id": 0, "Ev Idx": 6826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157269.585, "dur": 1027.400,
    "args": {
      "External id": 2462228,"Sequence number": 24740442, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6827
    }
  },
  {
    "ph": "s", "id": 206, "pid": 1336756, "tid": 1336756, "ts": 1514656157269.585,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157333.945, "dur": 43.303,
    "args": {
      "External id": 2462229,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157390.303, "dur": 101.946,
    "args": {
      "External id": 2462230,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157501.371, "dur": 56.699,
    "args": {
      "External id": 2462231,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157569.861, "dur": 34.545,
    "args": {
      "External id": 2462232,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157632.082, "dur": 26.441,
    "args": {
      "External id": 2462233,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157676.289, "dur": 15.484,
    "args": {
      "External id": 2462234,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157710.369, "dur": 131.285,
    "args": {
      "External id": 2462235,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157761.229, "dur": 10.449,
    "args": {
      "External id": 2462236,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157766.111, "dur": 4.738,
    "args": {
      "External id": 2462237,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157774.839, "dur": 5.514,
    "args": {
      "External id": 2462238,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157781.614, "dur": 1.164,
    "args": {
      "External id": 2462239,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157785.190, "dur": 6.135,
    "args": {
      "External id": 2462240,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157852.079, "dur": 45.938,
    "args": {
      "External id": 2462241,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157925.532, "dur": 29.671,
    "args": {
      "External id": 2462242,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656157964.828, "dur": 79.948,
    "args": {
      "External id": 2462243,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158057.383, "dur": 38.162,
    "args": {
      "External id": 2462244,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158118.946, "dur": 28.945,
    "args": {
      "External id": 2462245,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158153.539, "dur": 34.946,
    "args": {
      "External id": 2462246,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158209.069, "dur": 19.286,
    "args": {
      "External id": 2462247,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6846
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.10)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158373.160, "dur": 72.568,
    "args": {
      "External id": 2462248,"Record function id": 0, "Ev Idx": 6847
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158518.972, "dur": 68.134,
    "args": {
      "External id": 2462249,"Record function id": 0, "Ev Idx": 6848
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.11)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158597.681, "dur": 17798.765,
    "args": {
      "External id": 2462250,"Record function id": 0, "Ev Idx": 6849
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158607.162, "dur": 969.503,
    "args": {
      "External id": 2462251,"Record function id": 0, "Ev Idx": 6850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158690.652, "dur": 8.725,
    "args": {
      "External id": 2462252,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158712.754, "dur": 44.468,
    "args": {
      "External id": 2462253,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158718.930, "dur": 2.357,
    "args": {
      "External id": 2462254,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158726.383, "dur": 0.676,
    "args": {
      "External id": 2462255,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158729.191, "dur": 0.472,
    "args": {
      "External id": 2462256,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158731.770, "dur": 0.492,
    "args": {
      "External id": 2462257,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158736.079, "dur": 0.607,
    "args": {
      "External id": 2462258,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158738.243, "dur": 0.605,
    "args": {
      "External id": 2462259,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158740.751, "dur": 4.614,
    "args": {
      "External id": 2462260,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158747.186, "dur": 0.377,
    "args": {
      "External id": 2462261,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158749.254, "dur": 0.535,
    "args": {
      "External id": 2462262,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158773.791, "dur": 43.275,
    "args": {
      "External id": 2462263,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158848.858, "dur": 126.029,
    "args": {
      "External id": 2462264,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158862.512, "dur": 5.554,
    "args": {
      "External id": 2462265,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158873.137, "dur": 11.357,
    "args": {
      "External id": 2462266,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158877.866, "dur": 6.217,
    "args": {
      "External id": 2462267,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158881.989, "dur": 0.596,
    "args": {
      "External id": 2462268,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158891.305, "dur": 36.262,
    "args": {
      "External id": 2462269,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158893.556, "dur": 0.616,
    "args": {
      "External id": 2462270,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158895.575, "dur": 2.437,
    "args": {
      "External id": 2462271,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158900.082, "dur": 0.635,
    "args": {
      "External id": 2462272,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158902.579, "dur": 2.854,
    "args": {
      "External id": 2462273,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158909.715, "dur": 0.532,
    "args": {
      "External id": 2462274,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158911.517, "dur": 0.371,
    "args": {
      "External id": 2462275,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158913.805, "dur": 0.598,
    "args": {
      "External id": 2462276,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158918.231, "dur": 0.575,
    "args": {
      "External id": 2462277,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158920.522, "dur": 0.562,
    "args": {
      "External id": 2462278,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656158940.473, "dur": 26.394,
    "args": {
      "External id": 2462279,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159080.667, "dur": 381.055,
    "args": {
      "External id": 2462280,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159113.156, "dur": 343.492,
    "args": {
      "External id": 2462281,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6880, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159126.090, "dur": 324.768,
    "args": {
      "External id": 2462282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159489.910, "dur": 2.310,
    "args": {
      "External id": 2462283,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6882, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159600.097, "dur": 16604.148,
    "args": {
      "External id": 2462284,"Record function id": 0, "Ev Idx": 6883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159702.101, "dur": 6.290,
    "args": {
      "External id": 2462285,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159711.642, "dur": 1.226,
    "args": {
      "External id": 2462286,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159714.538, "dur": 3.420,
    "args": {
      "External id": 2462287,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159719.670, "dur": 1.063,
    "args": {
      "External id": 2462288,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159721.992, "dur": 1.031,
    "args": {
      "External id": 2462289,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159724.248, "dur": 1.271,
    "args": {
      "External id": 2462290,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159729.324, "dur": 1.125,
    "args": {
      "External id": 2462291,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159731.820, "dur": 2.272,
    "args": {
      "External id": 2462292,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159735.734, "dur": 0.896,
    "args": {
      "External id": 2462293,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159738.067, "dur": 0.669,
    "args": {
      "External id": 2462294,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159759.131, "dur": 16406.624,
    "args": {
      "External id": 2462295,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159775.980, "dur": 16382.571,
    "args": {
      "External id": 2462296,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159799.456, "dur": 15.171,
    "args": {
      "External id": 2462297,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159817.685, "dur": 16306.683,
    "args": {
      "External id": 2462298,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159820.138, "dur": 16303.699,
    "args": {
      "External id": 2462299,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159826.687, "dur": 6.604,
    "args": {
      "External id": 2462300,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656159834.838, "dur": 16286.073,
    "args": {
      "External id": 2462301,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176335.337, "dur": 36.324,
    "args": {
      "External id": 2462302,"Sequence number": 24740443, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6901
    }
  },
  {
    "ph": "s", "id": 205, "pid": 1336756, "tid": 1336756, "ts": 1514656176335.337,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176360.338, "dur": 6.877,
    "args": {
      "External id": 2462303,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176363.299, "dur": 3.692,
    "args": {
      "External id": 2462304,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176436.412, "dur": 78.267,
    "args": {
      "External id": 2462305,"Record function id": 0, "Ev Idx": 6904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176516.367, "dur": 1083.313,
    "args": {
      "External id": 2462306,"Record function id": 0, "Ev Idx": 6905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176573.495, "dur": 1013.116,
    "args": {
      "External id": 2462307,"Sequence number": 24740444, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6906
    }
  },
  {
    "ph": "s", "id": 204, "pid": 1336756, "tid": 1336756, "ts": 1514656176573.495,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176640.226, "dur": 41.881,
    "args": {
      "External id": 2462308,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176694.208, "dur": 102.425,
    "args": {
      "External id": 2462309,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176806.181, "dur": 39.007,
    "args": {
      "External id": 2462310,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176852.937, "dur": 30.620,
    "args": {
      "External id": 2462311,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176909.732, "dur": 26.395,
    "args": {
      "External id": 2462312,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656176953.614, "dur": 17.021,
    "args": {
      "External id": 2462313,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177023.300, "dur": 132.452,
    "args": {
      "External id": 2462314,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177073.904, "dur": 10.927,
    "args": {
      "External id": 2462315,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177078.644, "dur": 5.306,
    "args": {
      "External id": 2462316,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177087.628, "dur": 5.546,
    "args": {
      "External id": 2462317,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177094.561, "dur": 1.238,
    "args": {
      "External id": 2462318,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177098.307, "dur": 5.101,
    "args": {
      "External id": 2462319,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177166.963, "dur": 50.060,
    "args": {
      "External id": 2462320,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177248.982, "dur": 30.742,
    "args": {
      "External id": 2462321,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177288.347, "dur": 41.177,
    "args": {
      "External id": 2462322,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177338.713, "dur": 35.276,
    "args": {
      "External id": 2462323,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177394.042, "dur": 29.205,
    "args": {
      "External id": 2462324,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177428.748, "dur": 33.995,
    "args": {
      "External id": 2462325,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177484.466, "dur": 17.262,
    "args": {
      "External id": 2462326,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6925
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.11)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177663.451, "dur": 71.575,
    "args": {
      "External id": 2462327,"Record function id": 0, "Ev Idx": 6926
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177807.190, "dur": 45.849,
    "args": {
      "External id": 2462328,"Record function id": 0, "Ev Idx": 6927
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.12)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177862.746, "dur": 17933.202,
    "args": {
      "External id": 2462329,"Record function id": 0, "Ev Idx": 6928
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177871.230, "dur": 925.976,
    "args": {
      "External id": 2462330,"Record function id": 0, "Ev Idx": 6929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177949.651, "dur": 8.130,
    "args": {
      "External id": 2462331,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177970.840, "dur": 82.730,
    "args": {
      "External id": 2462332,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656177976.515, "dur": 2.303,
    "args": {
      "External id": 2462333,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178020.004, "dur": 0.771,
    "args": {
      "External id": 2462334,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178023.077, "dur": 0.216,
    "args": {
      "External id": 2462335,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178025.057, "dur": 0.388,
    "args": {
      "External id": 2462336,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178029.266, "dur": 0.858,
    "args": {
      "External id": 2462337,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178031.803, "dur": 0.770,
    "args": {
      "External id": 2462338,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178034.389, "dur": 4.794,
    "args": {
      "External id": 2462339,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178043.614, "dur": 0.570,
    "args": {
      "External id": 2462340,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178045.636, "dur": 0.390,
    "args": {
      "External id": 2462341,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178066.657, "dur": 45.583,
    "args": {
      "External id": 2462342,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178146.671, "dur": 120.934,
    "args": {
      "External id": 2462343,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178160.490, "dur": 5.074,
    "args": {
      "External id": 2462344,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178170.652, "dur": 13.139,
    "args": {
      "External id": 2462345,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178175.147, "dur": 8.176,
    "args": {
      "External id": 2462346,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178178.874, "dur": 2.911,
    "args": {
      "External id": 2462347,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178190.930, "dur": 32.464,
    "args": {
      "External id": 2462348,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178193.689, "dur": 0.229,
    "args": {
      "External id": 2462349,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178195.696, "dur": 0.547,
    "args": {
      "External id": 2462350,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178198.001, "dur": 0.475,
    "args": {
      "External id": 2462351,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178201.834, "dur": 2.440,
    "args": {
      "External id": 2462352,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178205.626, "dur": 0.385,
    "args": {
      "External id": 2462353,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178207.557, "dur": 2.889,
    "args": {
      "External id": 2462354,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178211.895, "dur": 0.428,
    "args": {
      "External id": 2462355,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178213.936, "dur": 0.362,
    "args": {
      "External id": 2462356,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178218.075, "dur": 0.402,
    "args": {
      "External id": 2462357,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178233.656, "dur": 25.631,
    "args": {
      "External id": 2462358,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178321.951, "dur": 380.423,
    "args": {
      "External id": 2462359,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178355.164, "dur": 341.941,
    "args": {
      "External id": 2462360,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6959, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178367.456, "dur": 323.723,
    "args": {
      "External id": 2462361,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178728.917, "dur": 2.334,
    "args": {
      "External id": 2462362,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6961, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178817.084, "dur": 16789.637,
    "args": {
      "External id": 2462363,"Record function id": 0, "Ev Idx": 6962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178913.536, "dur": 6.147,
    "args": {
      "External id": 2462364,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178922.996, "dur": 1.158,
    "args": {
      "External id": 2462365,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178926.203, "dur": 3.156,
    "args": {
      "External id": 2462366,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178931.032, "dur": 0.960,
    "args": {
      "External id": 2462367,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178933.773, "dur": 0.829,
    "args": {
      "External id": 2462368,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178936.068, "dur": 1.033,
    "args": {
      "External id": 2462369,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178940.797, "dur": 1.046,
    "args": {
      "External id": 2462370,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178943.483, "dur": 2.442,
    "args": {
      "External id": 2462371,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178947.570, "dur": 1.026,
    "args": {
      "External id": 2462372,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178950.198, "dur": 1.365,
    "args": {
      "External id": 2462373,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656178972.758, "dur": 16595.703,
    "args": {
      "External id": 2462374,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656179024.927, "dur": 16536.323,
    "args": {
      "External id": 2462375,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656179050.866, "dur": 14.638,
    "args": {
      "External id": 2462376,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656179068.934, "dur": 16447.933,
    "args": {
      "External id": 2462377,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656179071.295, "dur": 16445.104,
    "args": {
      "External id": 2462378,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656179077.645, "dur": 6.836,
    "args": {
      "External id": 2462379,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656179086.391, "dur": 16427.010,
    "args": {
      "External id": 2462380,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656195736.768, "dur": 35.193,
    "args": {
      "External id": 2462381,"Sequence number": 24740445, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6980
    }
  },
  {
    "ph": "s", "id": 203, "pid": 1336756, "tid": 1336756, "ts": 1514656195736.768,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656195760.304, "dur": 7.396,
    "args": {
      "External id": 2462382,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656195763.485, "dur": 3.985,
    "args": {
      "External id": 2462383,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656195837.017, "dur": 77.529,
    "args": {
      "External id": 2462384,"Record function id": 0, "Ev Idx": 6983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656195916.145, "dur": 1111.607,
    "args": {
      "External id": 2462385,"Record function id": 0, "Ev Idx": 6984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656195958.091, "dur": 1022.327,
    "args": {
      "External id": 2462386,"Sequence number": 24740446, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6985
    }
  },
  {
    "ph": "s", "id": 202, "pid": 1336756, "tid": 1336756, "ts": 1514656195958.091,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196059.841, "dur": 44.052,
    "args": {
      "External id": 2462387,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196116.610, "dur": 100.759,
    "args": {
      "External id": 2462388,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196226.999, "dur": 40.498,
    "args": {
      "External id": 2462389,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196276.416, "dur": 30.591,
    "args": {
      "External id": 2462390,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196333.483, "dur": 28.653,
    "args": {
      "External id": 2462391,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196381.096, "dur": 15.209,
    "args": {
      "External id": 2462392,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196416.444, "dur": 145.800,
    "args": {
      "External id": 2462393,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196464.385, "dur": 10.699,
    "args": {
      "External id": 2462394,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196469.275, "dur": 5.030,
    "args": {
      "External id": 2462395,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196478.019, "dur": 5.473,
    "args": {
      "External id": 2462396,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196484.812, "dur": 1.183,
    "args": {
      "External id": 2462397,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196488.559, "dur": 6.034,
    "args": {
      "External id": 2462398,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196574.430, "dur": 51.471,
    "args": {
      "External id": 2462399,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196657.214, "dur": 28.970,
    "args": {
      "External id": 2462400,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196697.385, "dur": 40.512,
    "args": {
      "External id": 2462401,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196746.379, "dur": 34.831,
    "args": {
      "External id": 2462402,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196808.651, "dur": 25.719,
    "args": {
      "External id": 2462403,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196841.807, "dur": 34.468,
    "args": {
      "External id": 2462404,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656196894.258, "dur": 19.288,
    "args": {
      "External id": 2462405,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7004
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.12)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197093.511, "dur": 75.640,
    "args": {
      "External id": 2462406,"Record function id": 0, "Ev Idx": 7005
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197243.624, "dur": 47.286,
    "args": {
      "External id": 2462407,"Record function id": 0, "Ev Idx": 7006
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.13)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197300.565, "dur": 18053.332,
    "args": {
      "External id": 2462408,"Record function id": 0, "Ev Idx": 7007
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197309.138, "dur": 958.241,
    "args": {
      "External id": 2462409,"Record function id": 0, "Ev Idx": 7008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197387.204, "dur": 8.195,
    "args": {
      "External id": 2462410,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197408.513, "dur": 41.353,
    "args": {
      "External id": 2462411,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197414.300, "dur": 2.629,
    "args": {
      "External id": 2462412,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197422.139, "dur": 0.225,
    "args": {
      "External id": 2462413,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197423.840, "dur": 0.506,
    "args": {
      "External id": 2462414,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197426.336, "dur": 0.405,
    "args": {
      "External id": 2462415,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197430.594, "dur": 0.411,
    "args": {
      "External id": 2462416,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197432.503, "dur": 0.404,
    "args": {
      "External id": 2462417,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197434.656, "dur": 4.435,
    "args": {
      "External id": 2462418,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197440.466, "dur": 0.407,
    "args": {
      "External id": 2462419,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197442.329, "dur": 0.562,
    "args": {
      "External id": 2462420,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197462.259, "dur": 42.623,
    "args": {
      "External id": 2462421,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197554.899, "dur": 128.349,
    "args": {
      "External id": 2462422,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197569.421, "dur": 4.803,
    "args": {
      "External id": 2462423,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197579.366, "dur": 11.188,
    "args": {
      "External id": 2462424,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197583.650, "dur": 6.478,
    "args": {
      "External id": 2462425,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197587.764, "dur": 0.857,
    "args": {
      "External id": 2462426,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197597.628, "dur": 35.765,
    "args": {
      "External id": 2462427,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197600.270, "dur": 2.583,
    "args": {
      "External id": 2462428,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197604.406, "dur": 0.524,
    "args": {
      "External id": 2462429,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197606.940, "dur": 0.603,
    "args": {
      "External id": 2462430,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197612.216, "dur": 2.742,
    "args": {
      "External id": 2462431,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197616.247, "dur": 0.547,
    "args": {
      "External id": 2462432,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197617.949, "dur": 0.576,
    "args": {
      "External id": 2462433,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197622.349, "dur": 0.590,
    "args": {
      "External id": 2462434,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197624.396, "dur": 0.359,
    "args": {
      "External id": 2462435,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197626.112, "dur": 2.372,
    "args": {
      "External id": 2462436,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197647.055, "dur": 28.285,
    "args": {
      "External id": 2462437,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197737.614, "dur": 428.881,
    "args": {
      "External id": 2462438,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197772.894, "dur": 387.642,
    "args": {
      "External id": 2462439,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7038, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656197782.833, "dur": 371.150,
    "args": {
      "External id": 2462440,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198194.070, "dur": 2.605,
    "args": {
      "External id": 2462441,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7040, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198288.585, "dur": 16859.266,
    "args": {
      "External id": 2462442,"Record function id": 0, "Ev Idx": 7041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198385.605, "dur": 6.205,
    "args": {
      "External id": 2462443,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198395.531, "dur": 1.232,
    "args": {
      "External id": 2462444,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198398.315, "dur": 3.175,
    "args": {
      "External id": 2462445,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198402.900, "dur": 1.236,
    "args": {
      "External id": 2462446,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198405.721, "dur": 1.223,
    "args": {
      "External id": 2462447,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198408.292, "dur": 1.076,
    "args": {
      "External id": 2462448,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198413.325, "dur": 0.913,
    "args": {
      "External id": 2462449,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198415.705, "dur": 3.225,
    "args": {
      "External id": 2462450,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198420.568, "dur": 0.579,
    "args": {
      "External id": 2462451,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198422.767, "dur": 0.889,
    "args": {
      "External id": 2462452,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198444.148, "dur": 16658.391,
    "args": {
      "External id": 2462453,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198460.436, "dur": 16632.861,
    "args": {
      "External id": 2462454,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198486.340, "dur": 14.267,
    "args": {
      "External id": 2462455,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198503.504, "dur": 16552.863,
    "args": {
      "External id": 2462456,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198506.039, "dur": 16549.540,
    "args": {
      "External id": 2462457,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198512.527, "dur": 4.975,
    "args": {
      "External id": 2462458,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656198519.300, "dur": 16532.773,
    "args": {
      "External id": 2462459,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215290.948, "dur": 37.344,
    "args": {
      "External id": 2462460,"Sequence number": 24740447, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7059
    }
  },
  {
    "ph": "s", "id": 201, "pid": 1336756, "tid": 1336756, "ts": 1514656215290.948,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215314.514, "dur": 9.257,
    "args": {
      "External id": 2462461,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215318.743, "dur": 4.738,
    "args": {
      "External id": 2462462,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215395.823, "dur": 75.004,
    "args": {
      "External id": 2462463,"Record function id": 0, "Ev Idx": 7062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215472.252, "dur": 1105.759,
    "args": {
      "External id": 2462464,"Record function id": 0, "Ev Idx": 7063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215513.099, "dur": 1050.454,
    "args": {
      "External id": 2462465,"Sequence number": 24740448, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7064
    }
  },
  {
    "ph": "s", "id": 200, "pid": 1336756, "tid": 1336756, "ts": 1514656215513.099,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215604.358, "dur": 46.274,
    "args": {
      "External id": 2462466,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215663.169, "dur": 104.543,
    "args": {
      "External id": 2462467,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215779.514, "dur": 38.729,
    "args": {
      "External id": 2462468,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215826.383, "dur": 30.600,
    "args": {
      "External id": 2462469,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215881.531, "dur": 25.528,
    "args": {
      "External id": 2462470,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215925.885, "dur": 16.166,
    "args": {
      "External id": 2462471,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656215961.042, "dur": 170.372,
    "args": {
      "External id": 2462472,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216048.430, "dur": 12.172,
    "args": {
      "External id": 2462473,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216054.217, "dur": 5.508,
    "args": {
      "External id": 2462474,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216063.492, "dur": 5.366,
    "args": {
      "External id": 2462475,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216070.345, "dur": 0.885,
    "args": {
      "External id": 2462476,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216073.530, "dur": 4.682,
    "args": {
      "External id": 2462477,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216142.994, "dur": 50.896,
    "args": {
      "External id": 2462478,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216225.818, "dur": 28.194,
    "args": {
      "External id": 2462479,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216262.673, "dur": 40.937,
    "args": {
      "External id": 2462480,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216311.680, "dur": 34.233,
    "args": {
      "External id": 2462481,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216370.525, "dur": 27.364,
    "args": {
      "External id": 2462482,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216403.474, "dur": 34.319,
    "args": {
      "External id": 2462483,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216457.136, "dur": 18.195,
    "args": {
      "External id": 2462484,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7083
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.13)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216643.507, "dur": 76.379,
    "args": {
      "External id": 2462485,"Record function id": 0, "Ev Idx": 7084
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216789.892, "dur": 44.035,
    "args": {
      "External id": 2462486,"Record function id": 0, "Ev Idx": 7085
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.14)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216843.934, "dur": 18067.723,
    "args": {
      "External id": 2462487,"Record function id": 0, "Ev Idx": 7086
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216855.919, "dur": 1019.665,
    "args": {
      "External id": 2462488,"Record function id": 0, "Ev Idx": 7087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216934.705, "dur": 8.046,
    "args": {
      "External id": 2462489,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216955.701, "dur": 90.693,
    "args": {
      "External id": 2462490,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216961.598, "dur": 2.182,
    "args": {
      "External id": 2462491,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216969.093, "dur": 0.234,
    "args": {
      "External id": 2462492,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216970.944, "dur": 0.590,
    "args": {
      "External id": 2462493,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216973.035, "dur": 0.638,
    "args": {
      "External id": 2462494,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216977.421, "dur": 0.602,
    "args": {
      "External id": 2462495,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656216979.583, "dur": 0.583,
    "args": {
      "External id": 2462496,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217024.853, "dur": 4.771,
    "args": {
      "External id": 2462497,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217035.949, "dur": 0.590,
    "args": {
      "External id": 2462498,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217037.857, "dur": 0.344,
    "args": {
      "External id": 2462499,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217062.362, "dur": 44.819,
    "args": {
      "External id": 2462500,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217141.016, "dur": 126.123,
    "args": {
      "External id": 2462501,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217156.151, "dur": 7.444,
    "args": {
      "External id": 2462502,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217168.816, "dur": 10.738,
    "args": {
      "External id": 2462503,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217172.993, "dur": 6.177,
    "args": {
      "External id": 2462504,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217177.049, "dur": 0.594,
    "args": {
      "External id": 2462505,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217187.721, "dur": 35.294,
    "args": {
      "External id": 2462506,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217190.314, "dur": 0.422,
    "args": {
      "External id": 2462507,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217192.811, "dur": 2.653,
    "args": {
      "External id": 2462508,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217197.101, "dur": 0.431,
    "args": {
      "External id": 2462509,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217199.122, "dur": 2.893,
    "args": {
      "External id": 2462510,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217205.812, "dur": 0.363,
    "args": {
      "External id": 2462511,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217208.094, "dur": 0.570,
    "args": {
      "External id": 2462512,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217210.058, "dur": 0.566,
    "args": {
      "External id": 2462513,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217214.609, "dur": 0.538,
    "args": {
      "External id": 2462514,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217216.401, "dur": 0.617,
    "args": {
      "External id": 2462515,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217234.294, "dur": 24.902,
    "args": {
      "External id": 2462516,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217321.853, "dur": 444.073,
    "args": {
      "External id": 2462517,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217355.522, "dur": 404.410,
    "args": {
      "External id": 2462518,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7117, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217365.973, "dur": 387.659,
    "args": {
      "External id": 2462519,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217795.712, "dur": 2.980,
    "args": {
      "External id": 2462520,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7119, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656217897.240, "dur": 16810.208,
    "args": {
      "External id": 2462521,"Record function id": 0, "Ev Idx": 7120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218037.746, "dur": 7.315,
    "args": {
      "External id": 2462522,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218048.881, "dur": 0.945,
    "args": {
      "External id": 2462523,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218051.475, "dur": 3.524,
    "args": {
      "External id": 2462524,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218056.732, "dur": 1.079,
    "args": {
      "External id": 2462525,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218059.113, "dur": 1.217,
    "args": {
      "External id": 2462526,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218061.460, "dur": 0.907,
    "args": {
      "External id": 2462527,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218066.444, "dur": 1.097,
    "args": {
      "External id": 2462528,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218068.899, "dur": 2.011,
    "args": {
      "External id": 2462529,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218072.594, "dur": 1.036,
    "args": {
      "External id": 2462530,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218075.212, "dur": 1.073,
    "args": {
      "External id": 2462531,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218099.412, "dur": 16565.475,
    "args": {
      "External id": 2462532,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218116.417, "dur": 16541.335,
    "args": {
      "External id": 2462533,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218143.518, "dur": 15.066,
    "args": {
      "External id": 2462534,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218161.501, "dur": 16460.066,
    "args": {
      "External id": 2462535,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218164.184, "dur": 16456.896,
    "args": {
      "External id": 2462536,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218170.577, "dur": 6.537,
    "args": {
      "External id": 2462537,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656218178.999, "dur": 16438.868,
    "args": {
      "External id": 2462538,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656234848.266, "dur": 38.157,
    "args": {
      "External id": 2462539,"Sequence number": 24740449, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7138
    }
  },
  {
    "ph": "s", "id": 199, "pid": 1336756, "tid": 1336756, "ts": 1514656234848.266,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656234874.062, "dur": 7.824,
    "args": {
      "External id": 2462540,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656234877.364, "dur": 4.288,
    "args": {
      "External id": 2462541,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656234953.482, "dur": 109.416,
    "args": {
      "External id": 2462542,"Record function id": 0, "Ev Idx": 7141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235066.106, "dur": 1109.464,
    "args": {
      "External id": 2462543,"Record function id": 0, "Ev Idx": 7142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235108.661, "dur": 1053.238,
    "args": {
      "External id": 2462544,"Sequence number": 24740450, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7143
    }
  },
  {
    "ph": "s", "id": 198, "pid": 1336756, "tid": 1336756, "ts": 1514656235108.661,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235177.216, "dur": 44.255,
    "args": {
      "External id": 2462545,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235234.339, "dur": 106.063,
    "args": {
      "External id": 2462546,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235349.845, "dur": 38.520,
    "args": {
      "External id": 2462547,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235396.779, "dur": 30.388,
    "args": {
      "External id": 2462548,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235452.427, "dur": 26.016,
    "args": {
      "External id": 2462549,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235496.740, "dur": 15.208,
    "args": {
      "External id": 2462550,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235544.307, "dur": 136.414,
    "args": {
      "External id": 2462551,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235597.037, "dur": 11.835,
    "args": {
      "External id": 2462552,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235602.563, "dur": 5.410,
    "args": {
      "External id": 2462553,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235612.240, "dur": 6.091,
    "args": {
      "External id": 2462554,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235619.911, "dur": 0.982,
    "args": {
      "External id": 2462555,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235623.560, "dur": 4.685,
    "args": {
      "External id": 2462556,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235697.179, "dur": 50.520,
    "args": {
      "External id": 2462557,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235783.180, "dur": 28.356,
    "args": {
      "External id": 2462558,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235820.215, "dur": 41.203,
    "args": {
      "External id": 2462559,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235869.267, "dur": 34.544,
    "args": {
      "External id": 2462560,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235924.618, "dur": 26.255,
    "args": {
      "External id": 2462561,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656235956.642, "dur": 82.344,
    "args": {
      "External id": 2462562,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236069.268, "dur": 24.513,
    "args": {
      "External id": 2462563,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7162
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.14)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236238.801, "dur": 75.002,
    "args": {
      "External id": 2462564,"Record function id": 0, "Ev Idx": 7163
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236386.305, "dur": 46.212,
    "args": {
      "External id": 2462565,"Record function id": 0, "Ev Idx": 7164
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.15)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236441.235, "dur": 18192.820,
    "args": {
      "External id": 2462566,"Record function id": 0, "Ev Idx": 7165
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236449.111, "dur": 947.574,
    "args": {
      "External id": 2462567,"Record function id": 0, "Ev Idx": 7166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236544.831, "dur": 9.923,
    "args": {
      "External id": 2462568,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236569.987, "dur": 39.812,
    "args": {
      "External id": 2462569,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236575.783, "dur": 2.448,
    "args": {
      "External id": 2462570,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236583.182, "dur": 0.215,
    "args": {
      "External id": 2462571,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236585.051, "dur": 0.587,
    "args": {
      "External id": 2462572,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236587.484, "dur": 0.810,
    "args": {
      "External id": 2462573,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236592.217, "dur": 0.374,
    "args": {
      "External id": 2462574,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236593.910, "dur": 0.209,
    "args": {
      "External id": 2462575,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236595.924, "dur": 2.616,
    "args": {
      "External id": 2462576,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236600.399, "dur": 0.435,
    "args": {
      "External id": 2462577,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236602.545, "dur": 0.551,
    "args": {
      "External id": 2462578,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236621.669, "dur": 44.690,
    "args": {
      "External id": 2462579,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236702.128, "dur": 131.260,
    "args": {
      "External id": 2462580,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236716.103, "dur": 3.851,
    "args": {
      "External id": 2462581,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236725.094, "dur": 10.411,
    "args": {
      "External id": 2462582,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236729.554, "dur": 5.556,
    "args": {
      "External id": 2462583,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236733.161, "dur": 0.561,
    "args": {
      "External id": 2462584,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236742.050, "dur": 36.126,
    "args": {
      "External id": 2462585,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236744.325, "dur": 3.428,
    "args": {
      "External id": 2462586,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236750.226, "dur": 0.441,
    "args": {
      "External id": 2462587,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236752.503, "dur": 0.821,
    "args": {
      "External id": 2462588,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236757.000, "dur": 2.377,
    "args": {
      "External id": 2462589,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236761.017, "dur": 0.181,
    "args": {
      "External id": 2462590,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236762.832, "dur": 0.583,
    "args": {
      "External id": 2462591,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236766.837, "dur": 0.528,
    "args": {
      "External id": 2462592,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236768.954, "dur": 0.391,
    "args": {
      "External id": 2462593,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236770.988, "dur": 2.492,
    "args": {
      "External id": 2462594,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236801.913, "dur": 23.143,
    "args": {
      "External id": 2462595,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236886.343, "dur": 412.559,
    "args": {
      "External id": 2462596,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236921.981, "dur": 371.497,
    "args": {
      "External id": 2462597,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7196, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656236934.947, "dur": 352.376,
    "args": {
      "External id": 2462598,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237326.536, "dur": 2.285,
    "args": {
      "External id": 2462599,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7198, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237417.577, "dur": 17013.583,
    "args": {
      "External id": 2462600,"Record function id": 0, "Ev Idx": 7199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237512.247, "dur": 6.342,
    "args": {
      "External id": 2462601,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237522.077, "dur": 1.537,
    "args": {
      "External id": 2462602,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237525.104, "dur": 20.286,
    "args": {
      "External id": 2462603,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237549.732, "dur": 1.148,
    "args": {
      "External id": 2462604,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237552.256, "dur": 0.832,
    "args": {
      "External id": 2462605,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237554.390, "dur": 0.940,
    "args": {
      "External id": 2462606,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237559.257, "dur": 1.021,
    "args": {
      "External id": 2462607,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237561.929, "dur": 1.956,
    "args": {
      "External id": 2462608,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237565.586, "dur": 0.861,
    "args": {
      "External id": 2462609,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237567.954, "dur": 1.167,
    "args": {
      "External id": 2462610,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237593.031, "dur": 16801.095,
    "args": {
      "External id": 2462611,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237609.628, "dur": 16777.972,
    "args": {
      "External id": 2462612,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237632.624, "dur": 13.519,
    "args": {
      "External id": 2462613,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237649.069, "dur": 16704.800,
    "args": {
      "External id": 2462614,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237651.805, "dur": 16701.555,
    "args": {
      "External id": 2462615,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237657.612, "dur": 6.154,
    "args": {
      "External id": 2462616,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656237665.239, "dur": 16684.886,
    "args": {
      "External id": 2462617,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254571.594, "dur": 37.594,
    "args": {
      "External id": 2462618,"Sequence number": 24740451, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7217
    }
  },
  {
    "ph": "s", "id": 197, "pid": 1336756, "tid": 1336756, "ts": 1514656254571.594,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254596.779, "dur": 7.709,
    "args": {
      "External id": 2462619,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254599.881, "dur": 4.249,
    "args": {
      "External id": 2462620,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254674.381, "dur": 77.599,
    "args": {
      "External id": 2462621,"Record function id": 0, "Ev Idx": 7220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254753.623, "dur": 1085.821,
    "args": {
      "External id": 2462622,"Record function id": 0, "Ev Idx": 7221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254797.117, "dur": 1029.191,
    "args": {
      "External id": 2462623,"Sequence number": 24740452, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7222
    }
  },
  {
    "ph": "s", "id": 196, "pid": 1336756, "tid": 1336756, "ts": 1514656254797.117,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254862.994, "dur": 42.878,
    "args": {
      "External id": 2462624,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656254918.779, "dur": 132.025,
    "args": {
      "External id": 2462625,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255064.351, "dur": 42.415,
    "args": {
      "External id": 2462626,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255114.782, "dur": 30.893,
    "args": {
      "External id": 2462627,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255172.910, "dur": 26.722,
    "args": {
      "External id": 2462628,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255218.045, "dur": 16.383,
    "args": {
      "External id": 2462629,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255253.437, "dur": 130.419,
    "args": {
      "External id": 2462630,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255303.543, "dur": 11.528,
    "args": {
      "External id": 2462631,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255308.999, "dur": 5.262,
    "args": {
      "External id": 2462632,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255317.903, "dur": 6.102,
    "args": {
      "External id": 2462633,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255325.235, "dur": 1.610,
    "args": {
      "External id": 2462634,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255329.438, "dur": 4.147,
    "args": {
      "External id": 2462635,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255394.835, "dur": 47.369,
    "args": {
      "External id": 2462636,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255473.219, "dur": 29.875,
    "args": {
      "External id": 2462637,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255513.096, "dur": 60.537,
    "args": {
      "External id": 2462638,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255585.159, "dur": 36.888,
    "args": {
      "External id": 2462639,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255647.932, "dur": 29.354,
    "args": {
      "External id": 2462640,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255682.462, "dur": 33.971,
    "args": {
      "External id": 2462641,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255736.524, "dur": 18.034,
    "args": {
      "External id": 2462642,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7241
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.15)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656255902.304, "dur": 73.479,
    "args": {
      "External id": 2462643,"Record function id": 0, "Ev Idx": 7242
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256090.918, "dur": 48.553,
    "args": {
      "External id": 2462644,"Record function id": 0, "Ev Idx": 7243
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.16)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256149.885, "dur": 18092.430,
    "args": {
      "External id": 2462645,"Record function id": 0, "Ev Idx": 7244
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256158.615, "dur": 801.110,
    "args": {
      "External id": 2462646,"Record function id": 0, "Ev Idx": 7245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256238.492, "dur": 9.161,
    "args": {
      "External id": 2462647,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256260.976, "dur": 42.768,
    "args": {
      "External id": 2462648,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256267.031, "dur": 2.579,
    "args": {
      "External id": 2462649,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256274.399, "dur": 0.408,
    "args": {
      "External id": 2462650,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256276.449, "dur": 0.621,
    "args": {
      "External id": 2462651,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256279.404, "dur": 0.367,
    "args": {
      "External id": 2462652,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256283.776, "dur": 0.648,
    "args": {
      "External id": 2462653,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256286.516, "dur": 0.506,
    "args": {
      "External id": 2462654,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256288.918, "dur": 4.296,
    "args": {
      "External id": 2462655,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256295.253, "dur": 0.492,
    "args": {
      "External id": 2462656,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256297.104, "dur": 0.535,
    "args": {
      "External id": 2462657,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256316.315, "dur": 40.833,
    "args": {
      "External id": 2462658,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256390.309, "dur": 119.193,
    "args": {
      "External id": 2462659,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256400.326, "dur": 3.600,
    "args": {
      "External id": 2462660,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256408.786, "dur": 13.856,
    "args": {
      "External id": 2462661,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256416.246, "dur": 5.955,
    "args": {
      "External id": 2462662,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256420.289, "dur": 0.689,
    "args": {
      "External id": 2462663,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256430.264, "dur": 36.908,
    "args": {
      "External id": 2462664,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256432.449, "dur": 3.721,
    "args": {
      "External id": 2462665,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256438.015, "dur": 0.389,
    "args": {
      "External id": 2462666,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256440.219, "dur": 0.331,
    "args": {
      "External id": 2462667,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256445.357, "dur": 1.893,
    "args": {
      "External id": 2462668,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256448.920, "dur": 0.384,
    "args": {
      "External id": 2462669,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256451.418, "dur": 0.366,
    "args": {
      "External id": 2462670,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256455.459, "dur": 0.388,
    "args": {
      "External id": 2462671,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256458.235, "dur": 0.389,
    "args": {
      "External id": 2462672,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256460.035, "dur": 2.745,
    "args": {
      "External id": 2462673,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256478.677, "dur": 22.844,
    "args": {
      "External id": 2462674,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256582.498, "dur": 290.809,
    "args": {
      "External id": 2462675,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256617.435, "dur": 250.915,
    "args": {
      "External id": 2462676,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7275, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256627.501, "dur": 234.982,
    "args": {
      "External id": 2462677,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256897.154, "dur": 2.043,
    "args": {
      "External id": 2462678,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7277, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656256979.401, "dur": 17069.769,
    "args": {
      "External id": 2462679,"Record function id": 0, "Ev Idx": 7278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257116.011, "dur": 6.706,
    "args": {
      "External id": 2462680,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257125.883, "dur": 1.260,
    "args": {
      "External id": 2462681,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257128.695, "dur": 2.579,
    "args": {
      "External id": 2462682,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257132.733, "dur": 0.696,
    "args": {
      "External id": 2462683,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257135.002, "dur": 0.643,
    "args": {
      "External id": 2462684,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257137.022, "dur": 1.174,
    "args": {
      "External id": 2462685,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257141.836, "dur": 1.178,
    "args": {
      "External id": 2462686,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257144.810, "dur": 2.400,
    "args": {
      "External id": 2462687,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257148.958, "dur": 0.777,
    "args": {
      "External id": 2462688,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257151.024, "dur": 0.739,
    "args": {
      "External id": 2462689,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257174.067, "dur": 16804.682,
    "args": {
      "External id": 2462690,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257190.377, "dur": 16781.199,
    "args": {
      "External id": 2462691,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257213.092, "dur": 13.584,
    "args": {
      "External id": 2462692,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257229.681, "dur": 16710.231,
    "args": {
      "External id": 2462693,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257232.129, "dur": 16707.333,
    "args": {
      "External id": 2462694,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257238.485, "dur": 5.648,
    "args": {
      "External id": 2462695,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656257245.708, "dur": 16690.723,
    "args": {
      "External id": 2462696,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274184.120, "dur": 33.998,
    "args": {
      "External id": 2462697,"Sequence number": 24740453, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7296
    }
  },
  {
    "ph": "s", "id": 195, "pid": 1336756, "tid": 1336756, "ts": 1514656274184.120,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274206.079, "dur": 7.623,
    "args": {
      "External id": 2462698,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274209.652, "dur": 3.752,
    "args": {
      "External id": 2462699,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274282.014, "dur": 75.443,
    "args": {
      "External id": 2462700,"Record function id": 0, "Ev Idx": 7299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274359.113, "dur": 1134.863,
    "args": {
      "External id": 2462701,"Record function id": 0, "Ev Idx": 7300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274399.908, "dur": 1081.043,
    "args": {
      "External id": 2462702,"Sequence number": 24740454, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7301
    }
  },
  {
    "ph": "s", "id": 194, "pid": 1336756, "tid": 1336756, "ts": 1514656274399.908,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274464.002, "dur": 44.119,
    "args": {
      "External id": 2462703,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274520.975, "dur": 123.009,
    "args": {
      "External id": 2462704,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274656.655, "dur": 39.545,
    "args": {
      "External id": 2462705,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274706.191, "dur": 30.605,
    "args": {
      "External id": 2462706,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274770.169, "dur": 31.188,
    "args": {
      "External id": 2462707,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274818.175, "dur": 15.858,
    "args": {
      "External id": 2462708,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274852.428, "dur": 169.631,
    "args": {
      "External id": 2462709,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274901.518, "dur": 11.522,
    "args": {
      "External id": 2462710,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274906.733, "dur": 5.592,
    "args": {
      "External id": 2462711,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274916.084, "dur": 6.069,
    "args": {
      "External id": 2462712,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274923.440, "dur": 1.444,
    "args": {
      "External id": 2462713,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656274927.704, "dur": 6.814,
    "args": {
      "External id": 2462714,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275035.738, "dur": 55.703,
    "args": {
      "External id": 2462715,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275127.049, "dur": 30.925,
    "args": {
      "External id": 2462716,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275167.326, "dur": 41.185,
    "args": {
      "External id": 2462717,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275214.967, "dur": 36.111,
    "args": {
      "External id": 2462718,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275275.085, "dur": 26.431,
    "args": {
      "External id": 2462719,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275309.185, "dur": 34.552,
    "args": {
      "External id": 2462720,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275389.995, "dur": 24.266,
    "args": {
      "External id": 2462721,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.16)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275575.085, "dur": 76.892,
    "args": {
      "External id": 2462722,"Record function id": 0, "Ev Idx": 7321
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275725.735, "dur": 44.834,
    "args": {
      "External id": 2462723,"Record function id": 0, "Ev Idx": 7322
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.17)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275779.292, "dur": 18513.455,
    "args": {
      "External id": 2462724,"Record function id": 0, "Ev Idx": 7323
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275788.183, "dur": 959.390,
    "args": {
      "External id": 2462725,"Record function id": 0, "Ev Idx": 7324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275871.007, "dur": 9.107,
    "args": {
      "External id": 2462726,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275893.676, "dur": 42.099,
    "args": {
      "External id": 2462727,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275899.667, "dur": 2.480,
    "args": {
      "External id": 2462728,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275907.602, "dur": 0.345,
    "args": {
      "External id": 2462729,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275909.980, "dur": 0.403,
    "args": {
      "External id": 2462730,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275912.376, "dur": 0.445,
    "args": {
      "External id": 2462731,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275916.191, "dur": 0.266,
    "args": {
      "External id": 2462732,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275918.035, "dur": 0.316,
    "args": {
      "External id": 2462733,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275920.075, "dur": 3.670,
    "args": {
      "External id": 2462734,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275925.958, "dur": 0.461,
    "args": {
      "External id": 2462735,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275928.075, "dur": 0.587,
    "args": {
      "External id": 2462736,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656275948.545, "dur": 75.918,
    "args": {
      "External id": 2462737,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276062.344, "dur": 124.117,
    "args": {
      "External id": 2462738,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276073.962, "dur": 5.132,
    "args": {
      "External id": 2462739,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276084.360, "dur": 10.797,
    "args": {
      "External id": 2462740,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276088.677, "dur": 6.093,
    "args": {
      "External id": 2462741,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276092.755, "dur": 0.553,
    "args": {
      "External id": 2462742,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276103.113, "dur": 36.393,
    "args": {
      "External id": 2462743,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276105.799, "dur": 2.788,
    "args": {
      "External id": 2462744,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276110.432, "dur": 0.538,
    "args": {
      "External id": 2462745,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276112.956, "dur": 0.549,
    "args": {
      "External id": 2462746,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276118.027, "dur": 2.510,
    "args": {
      "External id": 2462747,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276121.879, "dur": 0.521,
    "args": {
      "External id": 2462748,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276124.220, "dur": 0.414,
    "args": {
      "External id": 2462749,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276128.017, "dur": 0.381,
    "args": {
      "External id": 2462750,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276129.878, "dur": 0.189,
    "args": {
      "External id": 2462751,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276131.773, "dur": 2.889,
    "args": {
      "External id": 2462752,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276152.985, "dur": 25.466,
    "args": {
      "External id": 2462753,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276241.026, "dur": 408.260,
    "args": {
      "External id": 2462754,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276276.646, "dur": 366.920,
    "args": {
      "External id": 2462755,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7354, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276287.217, "dur": 348.303,
    "args": {
      "External id": 2462756,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276676.451, "dur": 2.463,
    "args": {
      "External id": 2462757,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7356, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276767.127, "dur": 17117.916,
    "args": {
      "External id": 2462758,"Record function id": 0, "Ev Idx": 7357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276870.343, "dur": 6.052,
    "args": {
      "External id": 2462759,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276879.888, "dur": 1.614,
    "args": {
      "External id": 2462760,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276883.247, "dur": 2.095,
    "args": {
      "External id": 2462761,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276887.113, "dur": 0.999,
    "args": {
      "External id": 2462762,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276889.469, "dur": 1.095,
    "args": {
      "External id": 2462763,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276891.958, "dur": 1.139,
    "args": {
      "External id": 2462764,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276897.242, "dur": 1.050,
    "args": {
      "External id": 2462765,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276899.822, "dur": 2.391,
    "args": {
      "External id": 2462766,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276903.897, "dur": 0.903,
    "args": {
      "External id": 2462767,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276906.393, "dur": 0.714,
    "args": {
      "External id": 2462768,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276928.412, "dur": 16818.051,
    "args": {
      "External id": 2462769,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276944.972, "dur": 16778.297,
    "args": {
      "External id": 2462770,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656276972.722, "dur": 49.153,
    "args": {
      "External id": 2462771,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656277026.346, "dur": 16623.839,
    "args": {
      "External id": 2462772,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656277028.712, "dur": 16619.241,
    "args": {
      "External id": 2462773,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656277035.736, "dur": 6.603,
    "args": {
      "External id": 2462774,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656277044.079, "dur": 16591.847,
    "args": {
      "External id": 2462775,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294183.227, "dur": 74.871,
    "args": {
      "External id": 2462776,"Sequence number": 24740455, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7375
    }
  },
  {
    "ph": "s", "id": 193, "pid": 1336756, "tid": 1336756, "ts": 1514656294183.227,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294233.048, "dur": 19.424,
    "args": {
      "External id": 2462777,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294242.471, "dur": 9.615,
    "args": {
      "External id": 2462778,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294367.518, "dur": 179.150,
    "args": {
      "External id": 2462779,"Record function id": 0, "Ev Idx": 7378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294550.219, "dur": 1376.263,
    "args": {
      "External id": 2462780,"Record function id": 0, "Ev Idx": 7379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294616.591, "dur": 1292.099,
    "args": {
      "External id": 2462781,"Sequence number": 24740456, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7380
    }
  },
  {
    "ph": "s", "id": 192, "pid": 1336756, "tid": 1336756, "ts": 1514656294616.591,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294726.797, "dur": 71.002,
    "args": {
      "External id": 2462782,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294816.786, "dur": 139.815,
    "args": {
      "External id": 2462783,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656294969.011, "dur": 73.638,
    "args": {
      "External id": 2462784,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295057.606, "dur": 35.270,
    "args": {
      "External id": 2462785,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295126.559, "dur": 33.034,
    "args": {
      "External id": 2462786,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295180.603, "dur": 16.494,
    "args": {
      "External id": 2462787,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295226.047, "dur": 181.813,
    "args": {
      "External id": 2462788,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295303.756, "dur": 15.858,
    "args": {
      "External id": 2462789,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295311.275, "dur": 7.276,
    "args": {
      "External id": 2462790,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295323.930, "dur": 8.424,
    "args": {
      "External id": 2462791,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295334.530, "dur": 1.573,
    "args": {
      "External id": 2462792,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295338.905, "dur": 6.455,
    "args": {
      "External id": 2462793,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295419.456, "dur": 48.021,
    "args": {
      "External id": 2462794,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295502.451, "dur": 50.984,
    "args": {
      "External id": 2462795,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295568.169, "dur": 46.787,
    "args": {
      "External id": 2462796,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295625.252, "dur": 34.972,
    "args": {
      "External id": 2462797,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295692.059, "dur": 36.594,
    "args": {
      "External id": 2462798,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295734.531, "dur": 34.985,
    "args": {
      "External id": 2462799,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656295792.374, "dur": 20.094,
    "args": {
      "External id": 2462800,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7399
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.17)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296058.829, "dur": 103.397,
    "args": {
      "External id": 2462801,"Record function id": 0, "Ev Idx": 7400
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296258.469, "dur": 51.585,
    "args": {
      "External id": 2462802,"Record function id": 0, "Ev Idx": 7401
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.18)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296320.604, "dur": 16350.785,
    "args": {
      "External id": 2462803,"Record function id": 0, "Ev Idx": 7402
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296329.218, "dur": 1145.828,
    "args": {
      "External id": 2462804,"Record function id": 0, "Ev Idx": 7403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296434.116, "dur": 8.986,
    "args": {
      "External id": 2462805,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296464.421, "dur": 49.665,
    "args": {
      "External id": 2462806,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296471.958, "dur": 2.783,
    "args": {
      "External id": 2462807,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296480.491, "dur": 0.603,
    "args": {
      "External id": 2462808,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296482.613, "dur": 0.567,
    "args": {
      "External id": 2462809,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296484.718, "dur": 0.740,
    "args": {
      "External id": 2462810,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296489.478, "dur": 0.521,
    "args": {
      "External id": 2462811,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296491.760, "dur": 0.488,
    "args": {
      "External id": 2462812,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296494.411, "dur": 5.385,
    "args": {
      "External id": 2462813,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296500.893, "dur": 0.431,
    "args": {
      "External id": 2462814,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296503.132, "dur": 0.370,
    "args": {
      "External id": 2462815,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296547.083, "dur": 50.365,
    "args": {
      "External id": 2462816,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296643.113, "dur": 140.773,
    "args": {
      "External id": 2462817,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296656.531, "dur": 5.079,
    "args": {
      "External id": 2462818,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296668.980, "dur": 13.921,
    "args": {
      "External id": 2462819,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296675.237, "dur": 7.281,
    "args": {
      "External id": 2462820,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296680.105, "dur": 0.717,
    "args": {
      "External id": 2462821,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296691.040, "dur": 41.989,
    "args": {
      "External id": 2462822,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296693.283, "dur": 2.566,
    "args": {
      "External id": 2462823,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296698.209, "dur": 0.491,
    "args": {
      "External id": 2462824,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296699.987, "dur": 0.289,
    "args": {
      "External id": 2462825,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296708.195, "dur": 2.296,
    "args": {
      "External id": 2462826,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296712.246, "dur": 0.563,
    "args": {
      "External id": 2462827,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296714.718, "dur": 0.477,
    "args": {
      "External id": 2462828,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296718.639, "dur": 0.506,
    "args": {
      "External id": 2462829,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296720.266, "dur": 0.548,
    "args": {
      "External id": 2462830,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296722.224, "dur": 3.089,
    "args": {
      "External id": 2462831,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296745.815, "dur": 28.997,
    "args": {
      "External id": 2462832,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296851.909, "dur": 513.843,
    "args": {
      "External id": 2462833,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296893.813, "dur": 466.063,
    "args": {
      "External id": 2462834,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7433, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656296908.140, "dur": 444.427,
    "args": {
      "External id": 2462835,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297396.392, "dur": 2.765,
    "args": {
      "External id": 2462836,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7435, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297498.606, "dur": 14947.118,
    "args": {
      "External id": 2462837,"Record function id": 0, "Ev Idx": 7436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297630.999, "dur": 6.598,
    "args": {
      "External id": 2462838,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297641.032, "dur": 1.658,
    "args": {
      "External id": 2462839,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297644.512, "dur": 4.353,
    "args": {
      "External id": 2462840,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297650.548, "dur": 1.063,
    "args": {
      "External id": 2462841,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297653.254, "dur": 1.189,
    "args": {
      "External id": 2462842,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297655.965, "dur": 1.192,
    "args": {
      "External id": 2462843,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297661.096, "dur": 1.085,
    "args": {
      "External id": 2462844,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297663.780, "dur": 4.008,
    "args": {
      "External id": 2462845,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297669.302, "dur": 1.163,
    "args": {
      "External id": 2462846,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297672.273, "dur": 0.725,
    "args": {
      "External id": 2462847,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297695.838, "dur": 14710.861,
    "args": {
      "External id": 2462848,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297714.814, "dur": 14685.071,
    "args": {
      "External id": 2462849,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297751.404, "dur": 16.714,
    "args": {
      "External id": 2462850,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297773.354, "dur": 14594.578,
    "args": {
      "External id": 2462851,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297776.845, "dur": 14590.618,
    "args": {
      "External id": 2462852,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297784.064, "dur": 6.552,
    "args": {
      "External id": 2462853,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656297793.389, "dur": 14571.239,
    "args": {
      "External id": 2462854,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312609.574, "dur": 35.237,
    "args": {
      "External id": 2462855,"Sequence number": 24740457, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7454
    }
  },
  {
    "ph": "s", "id": 191, "pid": 1336756, "tid": 1336756, "ts": 1514656312609.574,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312632.209, "dur": 7.392,
    "args": {
      "External id": 2462856,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312635.374, "dur": 3.860,
    "args": {
      "External id": 2462857,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312714.465, "dur": 78.706,
    "args": {
      "External id": 2462858,"Record function id": 0, "Ev Idx": 7457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312794.753, "dur": 1100.488,
    "args": {
      "External id": 2462859,"Record function id": 0, "Ev Idx": 7458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312839.997, "dur": 1042.036,
    "args": {
      "External id": 2462860,"Sequence number": 24740458, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7459
    }
  },
  {
    "ph": "s", "id": 190, "pid": 1336756, "tid": 1336756, "ts": 1514656312839.997,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312922.087, "dur": 42.752,
    "args": {
      "External id": 2462861,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656312979.036, "dur": 129.626,
    "args": {
      "External id": 2462862,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313123.664, "dur": 38.620,
    "args": {
      "External id": 2462863,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313168.787, "dur": 31.206,
    "args": {
      "External id": 2462864,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313231.233, "dur": 25.630,
    "args": {
      "External id": 2462865,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313273.335, "dur": 16.907,
    "args": {
      "External id": 2462866,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313309.298, "dur": 135.376,
    "args": {
      "External id": 2462867,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313359.579, "dur": 12.586,
    "args": {
      "External id": 2462868,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313365.648, "dur": 5.699,
    "args": {
      "External id": 2462869,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313374.931, "dur": 7.362,
    "args": {
      "External id": 2462870,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313383.791, "dur": 3.747,
    "args": {
      "External id": 2462871,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313389.807, "dur": 5.921,
    "args": {
      "External id": 2462872,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313455.365, "dur": 44.127,
    "args": {
      "External id": 2462873,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313548.277, "dur": 30.132,
    "args": {
      "External id": 2462874,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313590.721, "dur": 46.094,
    "args": {
      "External id": 2462875,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313643.790, "dur": 35.109,
    "args": {
      "External id": 2462876,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313706.707, "dur": 26.318,
    "args": {
      "External id": 2462877,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313738.260, "dur": 34.760,
    "args": {
      "External id": 2462878,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313790.338, "dur": 20.052,
    "args": {
      "External id": 2462879,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7478
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.18)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656313961.805, "dur": 114.945,
    "args": {
      "External id": 2462880,"Record function id": 0, "Ev Idx": 7479
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314155.966, "dur": 50.688,
    "args": {
      "External id": 2462881,"Record function id": 0, "Ev Idx": 7480
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.19)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314216.808, "dur": 18425.571,
    "args": {
      "External id": 2462882,"Record function id": 0, "Ev Idx": 7481
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314225.040, "dur": 996.454,
    "args": {
      "External id": 2462883,"Record function id": 0, "Ev Idx": 7482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314309.406, "dur": 8.984,
    "args": {
      "External id": 2462884,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314332.023, "dur": 42.877,
    "args": {
      "External id": 2462885,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314337.674, "dur": 2.374,
    "args": {
      "External id": 2462886,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314345.370, "dur": 0.637,
    "args": {
      "External id": 2462887,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314347.496, "dur": 0.737,
    "args": {
      "External id": 2462888,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314349.905, "dur": 0.335,
    "args": {
      "External id": 2462889,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314353.673, "dur": 0.430,
    "args": {
      "External id": 2462890,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314355.603, "dur": 0.273,
    "args": {
      "External id": 2462891,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314357.407, "dur": 4.701,
    "args": {
      "External id": 2462892,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314363.706, "dur": 0.390,
    "args": {
      "External id": 2462893,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314365.482, "dur": 0.334,
    "args": {
      "External id": 2462894,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314387.055, "dur": 41.509,
    "args": {
      "External id": 2462895,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314462.066, "dur": 148.978,
    "args": {
      "External id": 2462896,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314473.001, "dur": 4.131,
    "args": {
      "External id": 2462897,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314482.289, "dur": 10.784,
    "args": {
      "External id": 2462898,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314486.563, "dur": 6.121,
    "args": {
      "External id": 2462899,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314490.797, "dur": 0.603,
    "args": {
      "External id": 2462900,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314500.750, "dur": 57.964,
    "args": {
      "External id": 2462901,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314503.387, "dur": 2.895,
    "args": {
      "External id": 2462902,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314508.466, "dur": 0.554,
    "args": {
      "External id": 2462903,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314510.507, "dur": 0.524,
    "args": {
      "External id": 2462904,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314515.157, "dur": 2.705,
    "args": {
      "External id": 2462905,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314520.376, "dur": 0.459,
    "args": {
      "External id": 2462906,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314522.528, "dur": 0.463,
    "args": {
      "External id": 2462907,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314543.159, "dur": 0.747,
    "args": {
      "External id": 2462908,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314546.138, "dur": 0.402,
    "args": {
      "External id": 2462909,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314548.161, "dur": 3.642,
    "args": {
      "External id": 2462910,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314573.172, "dur": 29.246,
    "args": {
      "External id": 2462911,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314666.421, "dur": 447.348,
    "args": {
      "External id": 2462912,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314700.139, "dur": 407.985,
    "args": {
      "External id": 2462913,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7512, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656314710.165, "dur": 391.436,
    "args": {
      "External id": 2462914,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315142.007, "dur": 2.431,
    "args": {
      "External id": 2462915,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7514, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315242.280, "dur": 17181.908,
    "args": {
      "External id": 2462916,"Record function id": 0, "Ev Idx": 7515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315349.789, "dur": 6.568,
    "args": {
      "External id": 2462917,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315359.563, "dur": 1.239,
    "args": {
      "External id": 2462918,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315362.584, "dur": 4.888,
    "args": {
      "External id": 2462919,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315369.193, "dur": 1.166,
    "args": {
      "External id": 2462920,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315371.858, "dur": 0.931,
    "args": {
      "External id": 2462921,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315374.254, "dur": 1.056,
    "args": {
      "External id": 2462922,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315379.353, "dur": 1.095,
    "args": {
      "External id": 2462923,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315382.235, "dur": 3.500,
    "args": {
      "External id": 2462924,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315387.254, "dur": 0.977,
    "args": {
      "External id": 2462925,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315389.689, "dur": 1.031,
    "args": {
      "External id": 2462926,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315411.753, "dur": 16972.803,
    "args": {
      "External id": 2462927,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315428.138, "dur": 16949.131,
    "args": {
      "External id": 2462928,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315453.974, "dur": 14.087,
    "args": {
      "External id": 2462929,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315470.875, "dur": 16874.362,
    "args": {
      "External id": 2462930,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315477.222, "dur": 16867.503,
    "args": {
      "External id": 2462931,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315487.648, "dur": 6.254,
    "args": {
      "External id": 2462932,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656315495.638, "dur": 16845.575,
    "args": {
      "External id": 2462933,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332576.905, "dur": 39.859,
    "args": {
      "External id": 2462934,"Sequence number": 24740459, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7533
    }
  },
  {
    "ph": "s", "id": 189, "pid": 1336756, "tid": 1336756, "ts": 1514656332576.905,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332604.131, "dur": 7.976,
    "args": {
      "External id": 2462935,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332607.159, "dur": 4.546,
    "args": {
      "External id": 2462936,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332683.445, "dur": 73.668,
    "args": {
      "External id": 2462937,"Record function id": 0, "Ev Idx": 7536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332758.559, "dur": 1084.318,
    "args": {
      "External id": 2462938,"Record function id": 0, "Ev Idx": 7537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332797.007, "dur": 1032.746,
    "args": {
      "External id": 2462939,"Sequence number": 24740460, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7538
    }
  },
  {
    "ph": "s", "id": 188, "pid": 1336756, "tid": 1336756, "ts": 1514656332797.007,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332865.254, "dur": 41.843,
    "args": {
      "External id": 2462940,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656332919.021, "dur": 131.663,
    "args": {
      "External id": 2462941,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333065.793, "dur": 43.317,
    "args": {
      "External id": 2462942,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333118.812, "dur": 30.812,
    "args": {
      "External id": 2462943,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333177.735, "dur": 27.122,
    "args": {
      "External id": 2462944,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333224.358, "dur": 14.842,
    "args": {
      "External id": 2462945,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333260.443, "dur": 134.927,
    "args": {
      "External id": 2462946,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333312.584, "dur": 11.356,
    "args": {
      "External id": 2462947,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333317.944, "dur": 5.274,
    "args": {
      "External id": 2462948,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333326.640, "dur": 7.536,
    "args": {
      "External id": 2462949,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333335.631, "dur": 1.501,
    "args": {
      "External id": 2462950,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333339.532, "dur": 5.454,
    "args": {
      "External id": 2462951,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333405.704, "dur": 44.211,
    "args": {
      "External id": 2462952,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333478.705, "dur": 28.390,
    "args": {
      "External id": 2462953,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333516.640, "dur": 57.759,
    "args": {
      "External id": 2462954,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333586.149, "dur": 36.945,
    "args": {
      "External id": 2462955,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333647.866, "dur": 31.642,
    "args": {
      "External id": 2462956,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333684.993, "dur": 34.098,
    "args": {
      "External id": 2462957,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333740.072, "dur": 17.356,
    "args": {
      "External id": 2462958,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7557
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.19)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656333906.137, "dur": 73.556,
    "args": {
      "External id": 2462959,"Record function id": 0, "Ev Idx": 7558
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334091.933, "dur": 48.779,
    "args": {
      "External id": 2462960,"Record function id": 0, "Ev Idx": 7559
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.20)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334151.036, "dur": 18289.623,
    "args": {
      "External id": 2462961,"Record function id": 0, "Ev Idx": 7560
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334161.178, "dur": 927.560,
    "args": {
      "External id": 2462962,"Record function id": 0, "Ev Idx": 7561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334243.364, "dur": 9.091,
    "args": {
      "External id": 2462963,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334265.993, "dur": 43.453,
    "args": {
      "External id": 2462964,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334271.973, "dur": 2.435,
    "args": {
      "External id": 2462965,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334278.987, "dur": 0.750,
    "args": {
      "External id": 2462966,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334281.249, "dur": 0.576,
    "args": {
      "External id": 2462967,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334283.267, "dur": 0.638,
    "args": {
      "External id": 2462968,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334287.779, "dur": 0.325,
    "args": {
      "External id": 2462969,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334289.757, "dur": 0.588,
    "args": {
      "External id": 2462970,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334292.589, "dur": 4.212,
    "args": {
      "External id": 2462971,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334298.038, "dur": 0.510,
    "args": {
      "External id": 2462972,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334299.905, "dur": 0.434,
    "args": {
      "External id": 2462973,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334321.895, "dur": 42.484,
    "args": {
      "External id": 2462974,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334397.104, "dur": 122.690,
    "args": {
      "External id": 2462975,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334408.193, "dur": 4.556,
    "args": {
      "External id": 2462976,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334417.644, "dur": 10.553,
    "args": {
      "External id": 2462977,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334421.932, "dur": 5.844,
    "args": {
      "External id": 2462978,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334425.974, "dur": 0.607,
    "args": {
      "External id": 2462979,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334434.938, "dur": 36.032,
    "args": {
      "External id": 2462980,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334437.148, "dur": 2.650,
    "args": {
      "External id": 2462981,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334441.210, "dur": 0.682,
    "args": {
      "External id": 2462982,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334444.083, "dur": 0.510,
    "args": {
      "External id": 2462983,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334448.423, "dur": 2.685,
    "args": {
      "External id": 2462984,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334452.685, "dur": 0.299,
    "args": {
      "External id": 2462985,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334454.298, "dur": 0.593,
    "args": {
      "External id": 2462986,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334457.894, "dur": 0.507,
    "args": {
      "External id": 2462987,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334460.046, "dur": 0.591,
    "args": {
      "External id": 2462988,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334461.930, "dur": 2.747,
    "args": {
      "External id": 2462989,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334485.637, "dur": 25.469,
    "args": {
      "External id": 2462990,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334591.578, "dur": 364.913,
    "args": {
      "External id": 2462991,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334623.491, "dur": 327.738,
    "args": {
      "External id": 2462992,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7591, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656334634.643, "dur": 311.057,
    "args": {
      "External id": 2462993,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335013.896, "dur": 3.980,
    "args": {
      "External id": 2462994,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7593, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335109.365, "dur": 17129.164,
    "args": {
      "External id": 2462995,"Record function id": 0, "Ev Idx": 7594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335212.542, "dur": 6.297,
    "args": {
      "External id": 2462996,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335222.167, "dur": 1.312,
    "args": {
      "External id": 2462997,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335225.113, "dur": 3.087,
    "args": {
      "External id": 2462998,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335230.133, "dur": 1.289,
    "args": {
      "External id": 2462999,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335232.687, "dur": 1.104,
    "args": {
      "External id": 2463000,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335235.127, "dur": 1.060,
    "args": {
      "External id": 2463001,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335239.414, "dur": 0.953,
    "args": {
      "External id": 2463002,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335241.903, "dur": 3.939,
    "args": {
      "External id": 2463003,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335248.080, "dur": 0.778,
    "args": {
      "External id": 2463004,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335250.563, "dur": 0.851,
    "args": {
      "External id": 2463005,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335271.526, "dur": 16929.667,
    "args": {
      "External id": 2463006,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335287.164, "dur": 16907.550,
    "args": {
      "External id": 2463007,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335309.994, "dur": 15.428,
    "args": {
      "External id": 2463008,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335327.981, "dur": 16835.138,
    "args": {
      "External id": 2463009,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335330.240, "dur": 16832.332,
    "args": {
      "External id": 2463010,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335336.501, "dur": 5.312,
    "args": {
      "External id": 2463011,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656335343.517, "dur": 16816.139,
    "args": {
      "External id": 2463012,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352376.293, "dur": 41.833,
    "args": {
      "External id": 2463013,"Sequence number": 24740461, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7612
    }
  },
  {
    "ph": "s", "id": 187, "pid": 1336756, "tid": 1336756, "ts": 1514656352376.293,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352398.780, "dur": 10.290,
    "args": {
      "External id": 2463014,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352404.830, "dur": 3.940,
    "args": {
      "External id": 2463015,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352480.428, "dur": 93.974,
    "args": {
      "External id": 2463016,"Record function id": 0, "Ev Idx": 7615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352577.518, "dur": 1081.319,
    "args": {
      "External id": 2463017,"Record function id": 0, "Ev Idx": 7616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352621.002, "dur": 1024.486,
    "args": {
      "External id": 2463018,"Sequence number": 24740462, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7617
    }
  },
  {
    "ph": "s", "id": 186, "pid": 1336756, "tid": 1336756, "ts": 1514656352621.002,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352689.376, "dur": 43.625,
    "args": {
      "External id": 2463019,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352744.722, "dur": 101.511,
    "args": {
      "External id": 2463020,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352859.068, "dur": 37.610,
    "args": {
      "External id": 2463021,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352902.908, "dur": 32.583,
    "args": {
      "External id": 2463022,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656352962.348, "dur": 60.425,
    "args": {
      "External id": 2463023,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353042.312, "dur": 16.274,
    "args": {
      "External id": 2463024,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353079.450, "dur": 134.319,
    "args": {
      "External id": 2463025,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353128.728, "dur": 12.848,
    "args": {
      "External id": 2463026,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353134.759, "dur": 6.042,
    "args": {
      "External id": 2463027,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353144.293, "dur": 7.235,
    "args": {
      "External id": 2463028,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353153.470, "dur": 1.413,
    "args": {
      "External id": 2463029,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353159.674, "dur": 4.887,
    "args": {
      "External id": 2463030,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353224.557, "dur": 49.220,
    "args": {
      "External id": 2463031,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353304.513, "dur": 29.640,
    "args": {
      "External id": 2463032,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353344.956, "dur": 40.470,
    "args": {
      "External id": 2463033,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353392.437, "dur": 35.747,
    "args": {
      "External id": 2463034,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353451.659, "dur": 29.229,
    "args": {
      "External id": 2463035,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353486.313, "dur": 35.553,
    "args": {
      "External id": 2463036,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353556.112, "dur": 19.729,
    "args": {
      "External id": 2463037,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7636
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.20)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353721.202, "dur": 75.428,
    "args": {
      "External id": 2463038,"Record function id": 0, "Ev Idx": 7637
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353867.179, "dur": 45.386,
    "args": {
      "External id": 2463039,"Record function id": 0, "Ev Idx": 7638
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.21)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353922.488, "dur": 18341.980,
    "args": {
      "External id": 2463040,"Record function id": 0, "Ev Idx": 7639
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656353930.669, "dur": 903.594,
    "args": {
      "External id": 2463041,"Record function id": 0, "Ev Idx": 7640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354051.799, "dur": 9.376,
    "args": {
      "External id": 2463042,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354074.281, "dur": 42.874,
    "args": {
      "External id": 2463043,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354079.982, "dur": 2.391,
    "args": {
      "External id": 2463044,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354087.387, "dur": 0.633,
    "args": {
      "External id": 2463045,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354089.493, "dur": 0.851,
    "args": {
      "External id": 2463046,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354091.997, "dur": 0.540,
    "args": {
      "External id": 2463047,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354096.068, "dur": 0.560,
    "args": {
      "External id": 2463048,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354098.208, "dur": 0.245,
    "args": {
      "External id": 2463049,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354099.673, "dur": 4.545,
    "args": {
      "External id": 2463050,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354106.094, "dur": 0.453,
    "args": {
      "External id": 2463051,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354107.829, "dur": 0.319,
    "args": {
      "External id": 2463052,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354130.145, "dur": 41.642,
    "args": {
      "External id": 2463053,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354205.602, "dur": 118.531,
    "args": {
      "External id": 2463054,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354216.849, "dur": 3.435,
    "args": {
      "External id": 2463055,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354225.110, "dur": 10.392,
    "args": {
      "External id": 2463056,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354229.478, "dur": 5.632,
    "args": {
      "External id": 2463057,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354232.981, "dur": 0.802,
    "args": {
      "External id": 2463058,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354242.323, "dur": 36.110,
    "args": {
      "External id": 2463059,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354245.079, "dur": 2.455,
    "args": {
      "External id": 2463060,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354249.315, "dur": 0.567,
    "args": {
      "External id": 2463061,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354251.565, "dur": 0.496,
    "args": {
      "External id": 2463062,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354255.230, "dur": 3.469,
    "args": {
      "External id": 2463063,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354260.108, "dur": 0.518,
    "args": {
      "External id": 2463064,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354261.817, "dur": 0.530,
    "args": {
      "External id": 2463065,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354264.752, "dur": 0.233,
    "args": {
      "External id": 2463066,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354266.365, "dur": 0.364,
    "args": {
      "External id": 2463067,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354268.251, "dur": 2.792,
    "args": {
      "External id": 2463068,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354290.444, "dur": 25.172,
    "args": {
      "External id": 2463069,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354376.493, "dur": 363.675,
    "args": {
      "External id": 2463070,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354407.832, "dur": 327.129,
    "args": {
      "External id": 2463071,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7670, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354428.253, "dur": 300.982,
    "args": {
      "External id": 2463072,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354765.704, "dur": 2.413,
    "args": {
      "External id": 2463073,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7672, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354854.490, "dur": 17217.727,
    "args": {
      "External id": 2463074,"Record function id": 0, "Ev Idx": 7673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354952.201, "dur": 5.921,
    "args": {
      "External id": 2463075,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354961.399, "dur": 1.188,
    "args": {
      "External id": 2463076,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354964.332, "dur": 3.322,
    "args": {
      "External id": 2463077,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354969.446, "dur": 1.125,
    "args": {
      "External id": 2463078,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354972.018, "dur": 1.018,
    "args": {
      "External id": 2463079,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354974.431, "dur": 0.974,
    "args": {
      "External id": 2463080,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656354978.725, "dur": 0.773,
    "args": {
      "External id": 2463081,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355016.511, "dur": 3.411,
    "args": {
      "External id": 2463082,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355023.072, "dur": 0.836,
    "args": {
      "External id": 2463083,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355025.392, "dur": 0.933,
    "args": {
      "External id": 2463084,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355046.818, "dur": 16988.236,
    "args": {
      "External id": 2463085,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355062.377, "dur": 16965.994,
    "args": {
      "External id": 2463086,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355082.879, "dur": 15.050,
    "args": {
      "External id": 2463087,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355100.568, "dur": 16873.603,
    "args": {
      "External id": 2463088,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355103.133, "dur": 16870.559,
    "args": {
      "External id": 2463089,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355108.955, "dur": 5.798,
    "args": {
      "External id": 2463090,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656355116.597, "dur": 16854.196,
    "args": {
      "External id": 2463091,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372207.735, "dur": 33.462,
    "args": {
      "External id": 2463092,"Sequence number": 24740463, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7691
    }
  },
  {
    "ph": "s", "id": 185, "pid": 1336756, "tid": 1336756, "ts": 1514656372207.735,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372229.496, "dur": 6.850,
    "args": {
      "External id": 2463093,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372232.390, "dur": 3.722,
    "args": {
      "External id": 2463094,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372302.719, "dur": 74.468,
    "args": {
      "External id": 2463095,"Record function id": 0, "Ev Idx": 7694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372378.914, "dur": 1067.129,
    "args": {
      "External id": 2463096,"Record function id": 0, "Ev Idx": 7695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372418.969, "dur": 1014.958,
    "args": {
      "External id": 2463097,"Sequence number": 24740464, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7696
    }
  },
  {
    "ph": "s", "id": 184, "pid": 1336756, "tid": 1336756, "ts": 1514656372418.969,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372483.512, "dur": 39.571,
    "args": {
      "External id": 2463098,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372553.517, "dur": 104.845,
    "args": {
      "External id": 2463099,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372669.108, "dur": 36.996,
    "args": {
      "External id": 2463100,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372714.332, "dur": 30.683,
    "args": {
      "External id": 2463101,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372770.236, "dur": 27.487,
    "args": {
      "External id": 2463102,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372814.983, "dur": 14.435,
    "args": {
      "External id": 2463103,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372849.643, "dur": 130.798,
    "args": {
      "External id": 2463104,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372899.036, "dur": 11.077,
    "args": {
      "External id": 2463105,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372904.175, "dur": 5.116,
    "args": {
      "External id": 2463106,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372913.335, "dur": 6.868,
    "args": {
      "External id": 2463107,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372921.676, "dur": 1.386,
    "args": {
      "External id": 2463108,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656372925.650, "dur": 5.610,
    "args": {
      "External id": 2463109,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373029.069, "dur": 52.105,
    "args": {
      "External id": 2463110,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373113.301, "dur": 30.944,
    "args": {
      "External id": 2463111,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373153.273, "dur": 41.033,
    "args": {
      "External id": 2463112,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373202.895, "dur": 34.799,
    "args": {
      "External id": 2463113,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373261.547, "dur": 25.825,
    "args": {
      "External id": 2463114,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373292.627, "dur": 34.357,
    "args": {
      "External id": 2463115,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373345.202, "dur": 17.393,
    "args": {
      "External id": 2463116,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7715
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.21)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373507.621, "dur": 93.548,
    "args": {
      "External id": 2463117,"Record function id": 0, "Ev Idx": 7716
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373678.034, "dur": 47.227,
    "args": {
      "External id": 2463118,"Record function id": 0, "Ev Idx": 7717
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.22)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373735.830, "dur": 18182.413,
    "args": {
      "External id": 2463119,"Record function id": 0, "Ev Idx": 7718
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373744.012, "dur": 910.393,
    "args": {
      "External id": 2463120,"Record function id": 0, "Ev Idx": 7719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373827.799, "dur": 7.751,
    "args": {
      "External id": 2463121,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373847.958, "dur": 41.620,
    "args": {
      "External id": 2463122,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373853.480, "dur": 2.271,
    "args": {
      "External id": 2463123,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373860.152, "dur": 0.478,
    "args": {
      "External id": 2463124,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373861.972, "dur": 0.504,
    "args": {
      "External id": 2463125,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373863.800, "dur": 0.554,
    "args": {
      "External id": 2463126,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373867.862, "dur": 0.548,
    "args": {
      "External id": 2463127,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373869.531, "dur": 0.738,
    "args": {
      "External id": 2463128,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373871.732, "dur": 4.281,
    "args": {
      "External id": 2463129,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373877.643, "dur": 0.440,
    "args": {
      "External id": 2463130,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373879.676, "dur": 0.419,
    "args": {
      "External id": 2463131,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373900.826, "dur": 40.883,
    "args": {
      "External id": 2463132,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656373973.224, "dur": 162.226,
    "args": {
      "External id": 2463133,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374022.595, "dur": 6.155,
    "args": {
      "External id": 2463134,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374034.255, "dur": 10.470,
    "args": {
      "External id": 2463135,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374038.507, "dur": 5.805,
    "args": {
      "External id": 2463136,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374042.085, "dur": 0.613,
    "args": {
      "External id": 2463137,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374052.652, "dur": 35.214,
    "args": {
      "External id": 2463138,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374054.962, "dur": 2.270,
    "args": {
      "External id": 2463139,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374059.070, "dur": 0.492,
    "args": {
      "External id": 2463140,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374060.852, "dur": 0.552,
    "args": {
      "External id": 2463141,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374064.934, "dur": 2.795,
    "args": {
      "External id": 2463142,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374069.469, "dur": 0.511,
    "args": {
      "External id": 2463143,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374071.094, "dur": 0.547,
    "args": {
      "External id": 2463144,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374074.959, "dur": 0.311,
    "args": {
      "External id": 2463145,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374076.747, "dur": 0.422,
    "args": {
      "External id": 2463146,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374078.892, "dur": 1.987,
    "args": {
      "External id": 2463147,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374099.683, "dur": 27.354,
    "args": {
      "External id": 2463148,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374190.384, "dur": 370.179,
    "args": {
      "External id": 2463149,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374228.511, "dur": 326.746,
    "args": {
      "External id": 2463150,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7749, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374238.895, "dur": 309.702,
    "args": {
      "External id": 2463151,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374584.879, "dur": 2.843,
    "args": {
      "External id": 2463152,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7751, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374675.799, "dur": 17044.322,
    "args": {
      "External id": 2463153,"Record function id": 0, "Ev Idx": 7752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374774.999, "dur": 5.758,
    "args": {
      "External id": 2463154,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374784.245, "dur": 1.082,
    "args": {
      "External id": 2463155,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374787.032, "dur": 3.553,
    "args": {
      "External id": 2463156,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374792.310, "dur": 0.960,
    "args": {
      "External id": 2463157,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374794.662, "dur": 1.050,
    "args": {
      "External id": 2463158,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374798.964, "dur": 1.110,
    "args": {
      "External id": 2463159,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374801.843, "dur": 1.405,
    "args": {
      "External id": 2463160,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374805.035, "dur": 3.400,
    "args": {
      "External id": 2463161,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374810.063, "dur": 1.056,
    "args": {
      "External id": 2463162,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374813.960, "dur": 0.899,
    "args": {
      "External id": 2463163,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374833.466, "dur": 16847.090,
    "args": {
      "External id": 2463164,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374854.369, "dur": 16819.357,
    "args": {
      "External id": 2463165,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374876.354, "dur": 13.373,
    "args": {
      "External id": 2463166,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374892.749, "dur": 16749.103,
    "args": {
      "External id": 2463167,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374895.392, "dur": 16745.928,
    "args": {
      "External id": 2463168,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374901.525, "dur": 6.024,
    "args": {
      "External id": 2463169,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656374909.506, "dur": 16728.815,
    "args": {
      "External id": 2463170,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656391857.275, "dur": 34.787,
    "args": {
      "External id": 2463171,"Sequence number": 24740465, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7770
    }
  },
  {
    "ph": "s", "id": 183, "pid": 1336756, "tid": 1336756, "ts": 1514656391857.275,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656391879.069, "dur": 7.804,
    "args": {
      "External id": 2463172,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656391882.700, "dur": 3.960,
    "args": {
      "External id": 2463173,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656391957.849, "dur": 102.050,
    "args": {
      "External id": 2463174,"Record function id": 0, "Ev Idx": 7773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392062.842, "dur": 1078.357,
    "args": {
      "External id": 2463175,"Record function id": 0, "Ev Idx": 7774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392104.501, "dur": 1023.934,
    "args": {
      "External id": 2463176,"Sequence number": 24740466, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7775
    }
  },
  {
    "ph": "s", "id": 182, "pid": 1336756, "tid": 1336756, "ts": 1514656392104.501,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392171.911, "dur": 41.439,
    "args": {
      "External id": 2463177,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392226.340, "dur": 104.645,
    "args": {
      "External id": 2463178,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392341.508, "dur": 39.375,
    "args": {
      "External id": 2463179,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392389.518, "dur": 31.648,
    "args": {
      "External id": 2463180,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392446.664, "dur": 25.226,
    "args": {
      "External id": 2463181,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392488.748, "dur": 14.869,
    "args": {
      "External id": 2463182,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392522.332, "dur": 147.247,
    "args": {
      "External id": 2463183,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392586.942, "dur": 12.597,
    "args": {
      "External id": 2463184,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392592.413, "dur": 6.308,
    "args": {
      "External id": 2463185,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392602.276, "dur": 5.606,
    "args": {
      "External id": 2463186,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392609.386, "dur": 1.443,
    "args": {
      "External id": 2463187,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392613.105, "dur": 4.681,
    "args": {
      "External id": 2463188,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392681.065, "dur": 48.309,
    "args": {
      "External id": 2463189,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392760.933, "dur": 28.087,
    "args": {
      "External id": 2463190,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392799.149, "dur": 41.602,
    "args": {
      "External id": 2463191,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392849.499, "dur": 34.632,
    "args": {
      "External id": 2463192,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392907.163, "dur": 27.975,
    "args": {
      "External id": 2463193,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656392940.482, "dur": 34.821,
    "args": {
      "External id": 2463194,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393033.583, "dur": 20.301,
    "args": {
      "External id": 2463195,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7794
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.22)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393203.554, "dur": 73.740,
    "args": {
      "External id": 2463196,"Record function id": 0, "Ev Idx": 7795
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393348.015, "dur": 47.139,
    "args": {
      "External id": 2463197,"Record function id": 0, "Ev Idx": 7796
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.23)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393405.237, "dur": 18341.168,
    "args": {
      "External id": 2463198,"Record function id": 0, "Ev Idx": 7797
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393413.694, "dur": 894.693,
    "args": {
      "External id": 2463199,"Record function id": 0, "Ev Idx": 7798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393493.882, "dur": 7.769,
    "args": {
      "External id": 2463200,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393514.642, "dur": 59.627,
    "args": {
      "External id": 2463201,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393519.784, "dur": 2.281,
    "args": {
      "External id": 2463202,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393542.612, "dur": 1.041,
    "args": {
      "External id": 2463203,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393546.390, "dur": 0.546,
    "args": {
      "External id": 2463204,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393548.879, "dur": 0.590,
    "args": {
      "External id": 2463205,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393552.938, "dur": 0.645,
    "args": {
      "External id": 2463206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393555.093, "dur": 0.526,
    "args": {
      "External id": 2463207,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393557.239, "dur": 4.099,
    "args": {
      "External id": 2463208,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393563.409, "dur": 0.250,
    "args": {
      "External id": 2463209,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393565.098, "dur": 0.512,
    "args": {
      "External id": 2463210,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393587.514, "dur": 40.650,
    "args": {
      "External id": 2463211,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393663.231, "dur": 121.153,
    "args": {
      "External id": 2463212,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393675.050, "dur": 5.170,
    "args": {
      "External id": 2463213,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393685.366, "dur": 10.042,
    "args": {
      "External id": 2463214,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393689.636, "dur": 5.360,
    "args": {
      "External id": 2463215,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393693.319, "dur": 0.467,
    "args": {
      "External id": 2463216,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393703.149, "dur": 35.891,
    "args": {
      "External id": 2463217,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393705.617, "dur": 3.052,
    "args": {
      "External id": 2463218,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393710.331, "dur": 0.568,
    "args": {
      "External id": 2463219,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393712.418, "dur": 0.597,
    "args": {
      "External id": 2463220,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393716.724, "dur": 1.937,
    "args": {
      "External id": 2463221,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393720.126, "dur": 0.392,
    "args": {
      "External id": 2463222,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393722.308, "dur": 0.453,
    "args": {
      "External id": 2463223,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393726.010, "dur": 0.511,
    "args": {
      "External id": 2463224,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393728.047, "dur": 0.322,
    "args": {
      "External id": 2463225,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393729.999, "dur": 2.410,
    "args": {
      "External id": 2463226,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393752.024, "dur": 24.623,
    "args": {
      "External id": 2463227,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393840.028, "dur": 372.838,
    "args": {
      "External id": 2463228,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393871.214, "dur": 336.426,
    "args": {
      "External id": 2463229,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7828, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656393880.349, "dur": 321.611,
    "args": {
      "External id": 2463230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394236.787, "dur": 2.447,
    "args": {
      "External id": 2463231,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7830, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394329.049, "dur": 17212.188,
    "args": {
      "External id": 2463232,"Record function id": 0, "Ev Idx": 7831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394458.144, "dur": 6.129,
    "args": {
      "External id": 2463233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394468.029, "dur": 1.201,
    "args": {
      "External id": 2463234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394470.812, "dur": 3.368,
    "args": {
      "External id": 2463235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394477.686, "dur": 0.851,
    "args": {
      "External id": 2463236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394479.756, "dur": 0.808,
    "args": {
      "External id": 2463237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394481.910, "dur": 0.909,
    "args": {
      "External id": 2463238,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394484.386, "dur": 0.848,
    "args": {
      "External id": 2463239,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394488.247, "dur": 3.372,
    "args": {
      "External id": 2463240,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394493.052, "dur": 0.658,
    "args": {
      "External id": 2463241,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394495.221, "dur": 0.658,
    "args": {
      "External id": 2463242,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394517.157, "dur": 16973.441,
    "args": {
      "External id": 2463243,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394550.885, "dur": 16932.696,
    "args": {
      "External id": 2463244,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394565.929, "dur": 14.512,
    "args": {
      "External id": 2463245,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394585.486, "dur": 16866.421,
    "args": {
      "External id": 2463246,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394587.768, "dur": 16863.629,
    "args": {
      "External id": 2463247,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394593.383, "dur": 5.577,
    "args": {
      "External id": 2463248,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656394600.493, "dur": 16848.195,
    "args": {
      "External id": 2463249,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656411684.434, "dur": 37.417,
    "args": {
      "External id": 2463250,"Sequence number": 24740467, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7849
    }
  },
  {
    "ph": "s", "id": 181, "pid": 1336756, "tid": 1336756, "ts": 1514656411684.434,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656411709.394, "dur": 7.553,
    "args": {
      "External id": 2463251,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656411712.715, "dur": 3.993,
    "args": {
      "External id": 2463252,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656411784.562, "dur": 75.857,
    "args": {
      "External id": 2463253,"Record function id": 0, "Ev Idx": 7852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656411862.015, "dur": 1065.106,
    "args": {
      "External id": 2463254,"Record function id": 0, "Ev Idx": 7853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656411903.056, "dur": 1011.734,
    "args": {
      "External id": 2463255,"Sequence number": 24740468, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7854
    }
  },
  {
    "ph": "s", "id": 180, "pid": 1336756, "tid": 1336756, "ts": 1514656411903.056,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656411967.190, "dur": 71.555,
    "args": {
      "External id": 2463256,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412055.419, "dur": 101.552,
    "args": {
      "External id": 2463257,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412168.678, "dur": 37.518,
    "args": {
      "External id": 2463258,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412216.008, "dur": 31.275,
    "args": {
      "External id": 2463259,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412273.733, "dur": 24.475,
    "args": {
      "External id": 2463260,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412316.218, "dur": 14.761,
    "args": {
      "External id": 2463261,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412350.276, "dur": 126.121,
    "args": {
      "External id": 2463262,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412398.902, "dur": 11.489,
    "args": {
      "External id": 2463263,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412404.196, "dur": 5.398,
    "args": {
      "External id": 2463264,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412413.171, "dur": 5.396,
    "args": {
      "External id": 2463265,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412420.058, "dur": 1.171,
    "args": {
      "External id": 2463266,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412423.693, "dur": 4.902,
    "args": {
      "External id": 2463267,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412486.939, "dur": 59.564,
    "args": {
      "External id": 2463268,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412581.677, "dur": 29.484,
    "args": {
      "External id": 2463269,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412621.441, "dur": 44.628,
    "args": {
      "External id": 2463270,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412674.539, "dur": 35.100,
    "args": {
      "External id": 2463271,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412731.138, "dur": 25.373,
    "args": {
      "External id": 2463272,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412762.129, "dur": 36.048,
    "args": {
      "External id": 2463273,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656412820.273, "dur": 16.838,
    "args": {
      "External id": 2463274,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7873
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.23)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413026.910, "dur": 78.743,
    "args": {
      "External id": 2463275,"Record function id": 0, "Ev Idx": 7874
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413182.123, "dur": 46.535,
    "args": {
      "External id": 2463276,"Record function id": 0, "Ev Idx": 7875
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.24)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413238.359, "dur": 18342.974,
    "args": {
      "External id": 2463277,"Record function id": 0, "Ev Idx": 7876
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413248.331, "dur": 906.947,
    "args": {
      "External id": 2463278,"Record function id": 0, "Ev Idx": 7877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413329.647, "dur": 8.418,
    "args": {
      "External id": 2463279,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413350.604, "dur": 38.816,
    "args": {
      "External id": 2463280,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413355.974, "dur": 2.334,
    "args": {
      "External id": 2463281,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413362.409, "dur": 0.601,
    "args": {
      "External id": 2463282,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413364.690, "dur": 0.490,
    "args": {
      "External id": 2463283,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413366.547, "dur": 0.504,
    "args": {
      "External id": 2463284,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413369.987, "dur": 0.386,
    "args": {
      "External id": 2463285,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413371.993, "dur": 0.426,
    "args": {
      "External id": 2463286,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413373.900, "dur": 3.726,
    "args": {
      "External id": 2463287,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413378.939, "dur": 0.354,
    "args": {
      "External id": 2463288,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413380.856, "dur": 0.336,
    "args": {
      "External id": 2463289,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413400.769, "dur": 41.567,
    "args": {
      "External id": 2463290,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413474.668, "dur": 135.755,
    "args": {
      "External id": 2463291,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413485.573, "dur": 4.197,
    "args": {
      "External id": 2463292,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413494.827, "dur": 10.354,
    "args": {
      "External id": 2463293,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413499.147, "dur": 5.626,
    "args": {
      "External id": 2463294,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413502.835, "dur": 0.675,
    "args": {
      "External id": 2463295,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413511.945, "dur": 49.826,
    "args": {
      "External id": 2463296,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413514.386, "dur": 2.362,
    "args": {
      "External id": 2463297,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413518.289, "dur": 0.741,
    "args": {
      "External id": 2463298,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413520.487, "dur": 0.366,
    "args": {
      "External id": 2463299,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413523.927, "dur": 1.092,
    "args": {
      "External id": 2463300,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413526.319, "dur": 16.399,
    "args": {
      "External id": 2463301,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413545.535, "dur": 0.165,
    "args": {
      "External id": 2463302,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413549.449, "dur": 0.395,
    "args": {
      "External id": 2463303,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413550.877, "dur": 0.194,
    "args": {
      "External id": 2463304,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413552.686, "dur": 2.381,
    "args": {
      "External id": 2463305,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413575.524, "dur": 26.317,
    "args": {
      "External id": 2463306,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413664.741, "dur": 392.807,
    "args": {
      "External id": 2463307,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413695.227, "dur": 356.931,
    "args": {
      "External id": 2463308,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7907, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656413704.956, "dur": 341.515,
    "args": {
      "External id": 2463309,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414082.634, "dur": 2.566,
    "args": {
      "External id": 2463310,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7909, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414176.353, "dur": 17195.007,
    "args": {
      "External id": 2463311,"Record function id": 0, "Ev Idx": 7910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414274.445, "dur": 6.114,
    "args": {
      "External id": 2463312,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414283.988, "dur": 1.740,
    "args": {
      "External id": 2463313,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414287.381, "dur": 2.633,
    "args": {
      "External id": 2463314,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414291.713, "dur": 1.000,
    "args": {
      "External id": 2463315,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414304.017, "dur": 1.012,
    "args": {
      "External id": 2463316,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414310.750, "dur": 0.845,
    "args": {
      "External id": 2463317,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414315.252, "dur": 0.985,
    "args": {
      "External id": 2463318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414317.766, "dur": 3.335,
    "args": {
      "External id": 2463319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414322.463, "dur": 0.710,
    "args": {
      "External id": 2463320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414324.765, "dur": 0.602,
    "args": {
      "External id": 2463321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414347.250, "dur": 16986.835,
    "args": {
      "External id": 2463322,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414362.784, "dur": 16964.955,
    "args": {
      "External id": 2463323,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414381.000, "dur": 14.007,
    "args": {
      "External id": 2463324,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414397.893, "dur": 16898.964,
    "args": {
      "External id": 2463325,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414400.443, "dur": 16895.909,
    "args": {
      "External id": 2463326,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414405.736, "dur": 5.799,
    "args": {
      "External id": 2463327,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656414413.178, "dur": 16880.336,
    "args": {
      "External id": 2463328,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431508.647, "dur": 47.684,
    "args": {
      "External id": 2463329,"Sequence number": 24740469, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7928
    }
  },
  {
    "ph": "s", "id": 179, "pid": 1336756, "tid": 1336756, "ts": 1514656431508.647,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431543.454, "dur": 7.765,
    "args": {
      "External id": 2463330,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431546.560, "dur": 4.328,
    "args": {
      "External id": 2463331,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431620.651, "dur": 73.387,
    "args": {
      "External id": 2463332,"Record function id": 0, "Ev Idx": 7931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431695.435, "dur": 1049.197,
    "args": {
      "External id": 2463333,"Record function id": 0, "Ev Idx": 7932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431734.521, "dur": 997.242,
    "args": {
      "External id": 2463334,"Sequence number": 24740470, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7933
    }
  },
  {
    "ph": "s", "id": 178, "pid": 1336756, "tid": 1336756, "ts": 1514656431734.521,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431798.185, "dur": 39.584,
    "args": {
      "External id": 2463335,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431850.171, "dur": 101.673,
    "args": {
      "External id": 2463336,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656431960.197, "dur": 73.313,
    "args": {
      "External id": 2463337,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432046.010, "dur": 34.933,
    "args": {
      "External id": 2463338,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432106.037, "dur": 23.796,
    "args": {
      "External id": 2463339,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432149.532, "dur": 15.606,
    "args": {
      "External id": 2463340,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432182.304, "dur": 127.801,
    "args": {
      "External id": 2463341,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432234.693, "dur": 10.710,
    "args": {
      "External id": 2463342,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432239.455, "dur": 5.270,
    "args": {
      "External id": 2463343,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432248.047, "dur": 4.648,
    "args": {
      "External id": 2463344,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432254.047, "dur": 1.400,
    "args": {
      "External id": 2463345,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432257.500, "dur": 3.676,
    "args": {
      "External id": 2463346,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432320.857, "dur": 44.156,
    "args": {
      "External id": 2463347,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432394.837, "dur": 26.108,
    "args": {
      "External id": 2463348,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432429.650, "dur": 39.939,
    "args": {
      "External id": 2463349,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432478.553, "dur": 35.268,
    "args": {
      "External id": 2463350,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432553.393, "dur": 26.983,
    "args": {
      "External id": 2463351,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432587.384, "dur": 39.124,
    "args": {
      "External id": 2463352,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432646.795, "dur": 17.068,
    "args": {
      "External id": 2463353,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7952
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.24)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432809.314, "dur": 80.486,
    "args": {
      "External id": 2463354,"Record function id": 0, "Ev Idx": 7953
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656432962.306, "dur": 86.624,
    "args": {
      "External id": 2463355,"Record function id": 0, "Ev Idx": 7954
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.25)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433060.110, "dur": 18234.216,
    "args": {
      "External id": 2463356,"Record function id": 0, "Ev Idx": 7955
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433069.680, "dur": 910.282,
    "args": {
      "External id": 2463357,"Record function id": 0, "Ev Idx": 7956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433151.850, "dur": 9.042,
    "args": {
      "External id": 2463358,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433174.396, "dur": 44.567,
    "args": {
      "External id": 2463359,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433179.242, "dur": 2.292,
    "args": {
      "External id": 2463360,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433185.372, "dur": 0.729,
    "args": {
      "External id": 2463361,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433187.080, "dur": 0.772,
    "args": {
      "External id": 2463362,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433192.705, "dur": 5.870,
    "args": {
      "External id": 2463363,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433201.556, "dur": 0.749,
    "args": {
      "External id": 2463364,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433203.667, "dur": 0.389,
    "args": {
      "External id": 2463365,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433205.069, "dur": 2.007,
    "args": {
      "External id": 2463366,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433208.231, "dur": 0.405,
    "args": {
      "External id": 2463367,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433209.665, "dur": 0.361,
    "args": {
      "External id": 2463368,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433231.309, "dur": 39.888,
    "args": {
      "External id": 2463369,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433303.152, "dur": 110.999,
    "args": {
      "External id": 2463370,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433314.909, "dur": 4.259,
    "args": {
      "External id": 2463371,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433323.819, "dur": 10.155,
    "args": {
      "External id": 2463372,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433328.010, "dur": 5.578,
    "args": {
      "External id": 2463373,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433331.420, "dur": 0.718,
    "args": {
      "External id": 2463374,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433340.599, "dur": 30.385,
    "args": {
      "External id": 2463375,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433342.434, "dur": 2.258,
    "args": {
      "External id": 2463376,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433346.210, "dur": 0.599,
    "args": {
      "External id": 2463377,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433347.911, "dur": 0.747,
    "args": {
      "External id": 2463378,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433352.284, "dur": 0.667,
    "args": {
      "External id": 2463379,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433354.310, "dur": 0.523,
    "args": {
      "External id": 2463380,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433356.019, "dur": 0.355,
    "args": {
      "External id": 2463381,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433359.359, "dur": 0.504,
    "args": {
      "External id": 2463382,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433361.191, "dur": 0.307,
    "args": {
      "External id": 2463383,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433362.419, "dur": 2.064,
    "args": {
      "External id": 2463384,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433383.818, "dur": 21.592,
    "args": {
      "External id": 2463385,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433466.388, "dur": 415.283,
    "args": {
      "External id": 2463386,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433495.414, "dur": 380.919,
    "args": {
      "External id": 2463387,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7986, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433505.506, "dur": 362.573,
    "args": {
      "External id": 2463388,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656433905.179, "dur": 2.559,
    "args": {
      "External id": 2463389,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7988, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434037.710, "dur": 17062.129,
    "args": {
      "External id": 2463390,"Record function id": 0, "Ev Idx": 7989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434144.196, "dur": 6.798,
    "args": {
      "External id": 2463391,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434154.433, "dur": 1.314,
    "args": {
      "External id": 2463392,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434157.597, "dur": 1.668,
    "args": {
      "External id": 2463393,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434160.935, "dur": 1.085,
    "args": {
      "External id": 2463394,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434163.716, "dur": 1.101,
    "args": {
      "External id": 2463395,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434166.094, "dur": 1.511,
    "args": {
      "External id": 2463396,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434170.747, "dur": 1.252,
    "args": {
      "External id": 2463397,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434173.314, "dur": 3.404,
    "args": {
      "External id": 2463398,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434178.156, "dur": 1.110,
    "args": {
      "External id": 2463399,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434180.645, "dur": 0.936,
    "args": {
      "External id": 2463400,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434201.288, "dur": 16858.620,
    "args": {
      "External id": 2463401,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434217.425, "dur": 16835.910,
    "args": {
      "External id": 2463402,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434240.641, "dur": 12.068,
    "args": {
      "External id": 2463403,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434255.741, "dur": 16765.677,
    "args": {
      "External id": 2463404,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434258.300, "dur": 16762.553,
    "args": {
      "External id": 2463405,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434264.138, "dur": 5.965,
    "args": {
      "External id": 2463406,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656434271.761, "dur": 16745.998,
    "args": {
      "External id": 2463407,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451236.362, "dur": 33.812,
    "args": {
      "External id": 2463408,"Sequence number": 24740471, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8007
    }
  },
  {
    "ph": "s", "id": 177, "pid": 1336756, "tid": 1336756, "ts": 1514656451236.362,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451257.881, "dur": 7.465,
    "args": {
      "External id": 2463409,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451261.188, "dur": 3.993,
    "args": {
      "External id": 2463410,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451331.688, "dur": 71.246,
    "args": {
      "External id": 2463411,"Record function id": 0, "Ev Idx": 8010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451404.548, "dur": 1064.904,
    "args": {
      "External id": 2463412,"Record function id": 0, "Ev Idx": 8011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451443.525, "dur": 1013.285,
    "args": {
      "External id": 2463413,"Sequence number": 24740472, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8012
    }
  },
  {
    "ph": "s", "id": 176, "pid": 1336756, "tid": 1336756, "ts": 1514656451443.525,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451510.585, "dur": 55.698,
    "args": {
      "External id": 2463414,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451582.983, "dur": 104.170,
    "args": {
      "External id": 2463415,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451696.505, "dur": 38.276,
    "args": {
      "External id": 2463416,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451744.384, "dur": 31.114,
    "args": {
      "External id": 2463417,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451801.588, "dur": 25.062,
    "args": {
      "External id": 2463418,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451844.810, "dur": 14.742,
    "args": {
      "External id": 2463419,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451876.804, "dur": 162.179,
    "args": {
      "External id": 2463420,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451926.018, "dur": 11.911,
    "args": {
      "External id": 2463421,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451930.963, "dur": 6.051,
    "args": {
      "External id": 2463422,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451940.537, "dur": 4.834,
    "args": {
      "External id": 2463423,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451946.674, "dur": 1.369,
    "args": {
      "External id": 2463424,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656451950.340, "dur": 3.205,
    "args": {
      "External id": 2463425,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452052.176, "dur": 51.861,
    "args": {
      "External id": 2463426,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452136.212, "dur": 28.203,
    "args": {
      "External id": 2463427,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452172.933, "dur": 41.287,
    "args": {
      "External id": 2463428,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452222.820, "dur": 35.957,
    "args": {
      "External id": 2463429,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452281.326, "dur": 32.088,
    "args": {
      "External id": 2463430,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452319.756, "dur": 34.493,
    "args": {
      "External id": 2463431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452374.019, "dur": 17.210,
    "args": {
      "External id": 2463432,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8031
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.25)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452546.773, "dur": 77.388,
    "args": {
      "External id": 2463433,"Record function id": 0, "Ev Idx": 8032
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452696.963, "dur": 46.966,
    "args": {
      "External id": 2463434,"Record function id": 0, "Ev Idx": 8033
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.26)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452753.443, "dur": 18394.419,
    "args": {
      "External id": 2463435,"Record function id": 0, "Ev Idx": 8034
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452763.127, "dur": 880.437,
    "args": {
      "External id": 2463436,"Record function id": 0, "Ev Idx": 8035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452841.297, "dur": 8.318,
    "args": {
      "External id": 2463437,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452862.299, "dur": 38.368,
    "args": {
      "External id": 2463438,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452867.854, "dur": 2.434,
    "args": {
      "External id": 2463439,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452874.998, "dur": 0.844,
    "args": {
      "External id": 2463440,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452877.180, "dur": 0.710,
    "args": {
      "External id": 2463441,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452879.126, "dur": 0.795,
    "args": {
      "External id": 2463442,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452882.819, "dur": 0.489,
    "args": {
      "External id": 2463443,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452884.385, "dur": 0.781,
    "args": {
      "External id": 2463444,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452886.241, "dur": 2.457,
    "args": {
      "External id": 2463445,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452889.892, "dur": 0.771,
    "args": {
      "External id": 2463446,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452891.763, "dur": 0.638,
    "args": {
      "External id": 2463447,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656452911.955, "dur": 39.562,
    "args": {
      "External id": 2463448,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453018.385, "dur": 127.165,
    "args": {
      "External id": 2463449,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453031.516, "dur": 6.252,
    "args": {
      "External id": 2463450,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453043.606, "dur": 10.480,
    "args": {
      "External id": 2463451,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453047.619, "dur": 6.062,
    "args": {
      "External id": 2463452,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453051.365, "dur": 0.824,
    "args": {
      "External id": 2463453,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453061.906, "dur": 36.776,
    "args": {
      "External id": 2463454,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453063.931, "dur": 2.361,
    "args": {
      "External id": 2463455,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453067.686, "dur": 0.532,
    "args": {
      "External id": 2463456,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453069.378, "dur": 0.596,
    "args": {
      "External id": 2463457,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453073.192, "dur": 0.740,
    "args": {
      "External id": 2463458,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453075.109, "dur": 0.438,
    "args": {
      "External id": 2463459,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453080.149, "dur": 3.070,
    "args": {
      "External id": 2463460,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453086.049, "dur": 0.853,
    "args": {
      "External id": 2463461,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453088.042, "dur": 0.639,
    "args": {
      "External id": 2463462,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453089.704, "dur": 2.533,
    "args": {
      "External id": 2463463,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453110.428, "dur": 26.162,
    "args": {
      "External id": 2463464,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453198.050, "dur": 351.607,
    "args": {
      "External id": 2463465,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453227.684, "dur": 316.368,
    "args": {
      "External id": 2463466,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8065, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453237.138, "dur": 285.044,
    "args": {
      "External id": 2463467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453572.201, "dur": 2.762,
    "args": {
      "External id": 2463468,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8067, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453664.090, "dur": 17261.647,
    "args": {
      "External id": 2463469,"Record function id": 0, "Ev Idx": 8068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453762.150, "dur": 6.132,
    "args": {
      "External id": 2463470,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453771.640, "dur": 1.222,
    "args": {
      "External id": 2463471,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453774.429, "dur": 1.095,
    "args": {
      "External id": 2463472,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453777.115, "dur": 1.421,
    "args": {
      "External id": 2463473,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453779.999, "dur": 1.350,
    "args": {
      "External id": 2463474,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453782.571, "dur": 1.360,
    "args": {
      "External id": 2463475,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453787.897, "dur": 1.158,
    "args": {
      "External id": 2463476,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453790.433, "dur": 6.129,
    "args": {
      "External id": 2463477,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453797.907, "dur": 1.117,
    "args": {
      "External id": 2463478,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453800.290, "dur": 1.604,
    "args": {
      "External id": 2463479,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453820.239, "dur": 17067.643,
    "args": {
      "External id": 2463480,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453839.637, "dur": 17041.364,
    "args": {
      "External id": 2463481,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453856.204, "dur": 12.849,
    "args": {
      "External id": 2463482,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453871.984, "dur": 16978.351,
    "args": {
      "External id": 2463483,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453874.204, "dur": 16975.544,
    "args": {
      "External id": 2463484,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453879.770, "dur": 5.306,
    "args": {
      "External id": 2463485,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656453886.632, "dur": 16960.061,
    "args": {
      "External id": 2463486,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471087.355, "dur": 35.296,
    "args": {
      "External id": 2463487,"Sequence number": 24740473, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8086
    }
  },
  {
    "ph": "s", "id": 175, "pid": 1336756, "tid": 1336756, "ts": 1514656471087.355,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471109.247, "dur": 8.329,
    "args": {
      "External id": 2463488,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471112.663, "dur": 4.475,
    "args": {
      "External id": 2463489,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471187.708, "dur": 73.345,
    "args": {
      "External id": 2463490,"Record function id": 0, "Ev Idx": 8089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471262.504, "dur": 1070.823,
    "args": {
      "External id": 2463491,"Record function id": 0, "Ev Idx": 8090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471301.308, "dur": 1019.238,
    "args": {
      "External id": 2463492,"Sequence number": 24740474, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8091
    }
  },
  {
    "ph": "s", "id": 174, "pid": 1336756, "tid": 1336756, "ts": 1514656471301.308,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471365.765, "dur": 41.495,
    "args": {
      "External id": 2463493,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471419.226, "dur": 105.471,
    "args": {
      "External id": 2463494,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471552.221, "dur": 43.605,
    "args": {
      "External id": 2463495,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471606.263, "dur": 31.802,
    "args": {
      "External id": 2463496,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471664.521, "dur": 27.567,
    "args": {
      "External id": 2463497,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471709.890, "dur": 15.510,
    "args": {
      "External id": 2463498,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471741.943, "dur": 127.181,
    "args": {
      "External id": 2463499,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471793.184, "dur": 10.834,
    "args": {
      "External id": 2463500,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471797.808, "dur": 5.477,
    "args": {
      "External id": 2463501,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471806.725, "dur": 4.718,
    "args": {
      "External id": 2463502,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471812.673, "dur": 1.482,
    "args": {
      "External id": 2463503,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471816.370, "dur": 3.397,
    "args": {
      "External id": 2463504,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471881.131, "dur": 44.805,
    "args": {
      "External id": 2463505,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656471956.199, "dur": 64.299,
    "args": {
      "External id": 2463506,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472032.595, "dur": 46.428,
    "args": {
      "External id": 2463507,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472087.932, "dur": 35.626,
    "args": {
      "External id": 2463508,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472147.969, "dur": 25.673,
    "args": {
      "External id": 2463509,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472179.475, "dur": 35.699,
    "args": {
      "External id": 2463510,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472234.973, "dur": 17.885,
    "args": {
      "External id": 2463511,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8110
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.26)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472396.092, "dur": 75.349,
    "args": {
      "External id": 2463512,"Record function id": 0, "Ev Idx": 8111
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472562.428, "dur": 50.152,
    "args": {
      "External id": 2463513,"Record function id": 0, "Ev Idx": 8112
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.27)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472623.160, "dur": 18394.334,
    "args": {
      "External id": 2463514,"Record function id": 0, "Ev Idx": 8113
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.27)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472631.975, "dur": 873.269,
    "args": {
      "External id": 2463515,"Record function id": 0, "Ev Idx": 8114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472716.441, "dur": 9.418,
    "args": {
      "External id": 2463516,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472738.972, "dur": 36.367,
    "args": {
      "External id": 2463517,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472743.811, "dur": 2.356,
    "args": {
      "External id": 2463518,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472750.332, "dur": 0.848,
    "args": {
      "External id": 2463519,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472751.846, "dur": 1.011,
    "args": {
      "External id": 2463520,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472753.725, "dur": 0.663,
    "args": {
      "External id": 2463521,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472757.507, "dur": 0.713,
    "args": {
      "External id": 2463522,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472759.400, "dur": 0.429,
    "args": {
      "External id": 2463523,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472760.867, "dur": 2.639,
    "args": {
      "External id": 2463524,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472764.705, "dur": 0.720,
    "args": {
      "External id": 2463525,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472766.352, "dur": 0.657,
    "args": {
      "External id": 2463526,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472786.670, "dur": 37.713,
    "args": {
      "External id": 2463527,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472856.866, "dur": 105.064,
    "args": {
      "External id": 2463528,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "3", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472867.613, "dur": 4.691,
    "args": {
      "External id": 2463529,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472876.860, "dur": 10.146,
    "args": {
      "External id": 2463530,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472880.914, "dur": 5.676,
    "args": {
      "External id": 2463531,"Record function id": 0, "Concrete Inputs": ["", "0", "19269120", "25692160", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472884.211, "dur": 1.133,
    "args": {
      "External id": 2463532,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472893.008, "dur": 28.556,
    "args": {
      "External id": 2463533,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472894.634, "dur": 2.139,
    "args": {
      "External id": 2463534,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "19269120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472898.108, "dur": 0.844,
    "args": {
      "External id": 2463535,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19269376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472899.772, "dur": 0.814,
    "args": {
      "External id": 2463536,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "19793664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472903.463, "dur": 0.552,
    "args": {
      "External id": 2463537,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20317952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472904.951, "dur": 0.684,
    "args": {
      "External id": 2463538,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "20842240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472906.804, "dur": 0.842,
    "args": {
      "External id": 2463539,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21366528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472910.039, "dur": 0.628,
    "args": {
      "External id": 2463540,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "21366784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472911.757, "dur": 0.646,
    "args": {
      "External id": 2463541,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "22808576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472913.330, "dur": 1.717,
    "args": {
      "External id": 2463542,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24250368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656472933.359, "dur": 20.662,
    "args": {
      "External id": 2463543,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473055.005, "dur": 357.818,
    "args": {
      "External id": 2463544,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473086.350, "dur": 321.525,
    "args": {
      "External id": 2463545,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 3, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8144, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473096.637, "dur": 305.455,
    "args": {
      "External id": 2463546,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473434.696, "dur": 2.556,
    "args": {
      "External id": 2463547,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8146, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.27)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473525.288, "dur": 17270.464,
    "args": {
      "External id": 2463548,"Record function id": 0, "Ev Idx": 8147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473643.758, "dur": 6.745,
    "args": {
      "External id": 2463549,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473654.017, "dur": 1.498,
    "args": {
      "External id": 2463550,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473657.336, "dur": 1.472,
    "args": {
      "External id": 2463551,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473670.803, "dur": 1.420,
    "args": {
      "External id": 2463552,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473673.682, "dur": 1.416,
    "args": {
      "External id": 2463553,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473676.471, "dur": 1.124,
    "args": {
      "External id": 2463554,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473680.963, "dur": 0.994,
    "args": {
      "External id": 2463555,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473683.335, "dur": 3.805,
    "args": {
      "External id": 2463556,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473688.368, "dur": 1.078,
    "args": {
      "External id": 2463557,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473690.908, "dur": 0.819,
    "args": {
      "External id": 2463558,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473712.934, "dur": 17044.478,
    "args": {
      "External id": 2463559,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473728.714, "dur": 17022.167,
    "args": {
      "External id": 2463560,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473746.538, "dur": 12.904,
    "args": {
      "External id": 2463561,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473762.463, "dur": 16957.632,
    "args": {
      "External id": 2463562,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473764.960, "dur": 16954.639,
    "args": {
      "External id": 2463563,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473770.436, "dur": 5.639,
    "args": {
      "External id": 2463564,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656473777.701, "dur": 16938.898,
    "args": {
      "External id": 2463565,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656490929.662, "dur": 36.778,
    "args": {
      "External id": 2463566,"Sequence number": 24740475, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8165
    }
  },
  {
    "ph": "s", "id": 173, "pid": 1336756, "tid": 1336756, "ts": 1514656490929.662,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656490954.141, "dur": 7.731,
    "args": {
      "External id": 2463567,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656490957.451, "dur": 4.156,
    "args": {
      "External id": 2463568,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491056.745, "dur": 73.207,
    "args": {
      "External id": 2463569,"Record function id": 0, "Ev Idx": 8168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491131.521, "dur": 1074.536,
    "args": {
      "External id": 2463570,"Record function id": 0, "Ev Idx": 8169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491171.769, "dur": 1021.694,
    "args": {
      "External id": 2463571,"Sequence number": 24740476, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8170
    }
  },
  {
    "ph": "s", "id": 172, "pid": 1336756, "tid": 1336756, "ts": 1514656491171.769,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491243.112, "dur": 40.037,
    "args": {
      "External id": 2463572,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491295.950, "dur": 106.397,
    "args": {
      "External id": 2463573,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491412.072, "dur": 39.014,
    "args": {
      "External id": 2463574,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491460.234, "dur": 32.520,
    "args": {
      "External id": 2463575,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491518.446, "dur": 43.457,
    "args": {
      "External id": 2463576,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491582.915, "dur": 16.064,
    "args": {
      "External id": 2463577,"kernel_hash": "cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/ah/cahe5odluo4swiyj4n2lohwh57qg5lym5qsijyjsjca7pvto5oka.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491616.178, "dur": 126.628,
    "args": {
      "External id": 2463578,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491665.786, "dur": 11.280,
    "args": {
      "External id": 2463579,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491670.439, "dur": 5.889,
    "args": {
      "External id": 2463580,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491679.552, "dur": 4.736,
    "args": {
      "External id": 2463581,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491685.612, "dur": 1.901,
    "args": {
      "External id": 2463582,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491689.718, "dur": 4.100,
    "args": {
      "External id": 2463583,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491754.465, "dur": 47.608,
    "args": {
      "External id": 2463584,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491833.491, "dur": 27.603,
    "args": {
      "External id": 2463585,"kernel_hash": "ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/kb/ckbsutlugbs4qoncxuwq6kkwome3xojo3ke4p7z74lcl5h4q62to.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491868.894, "dur": 41.356,
    "args": {
      "External id": 2463586,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491919.505, "dur": 35.986,
    "args": {
      "External id": 2463587,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656491977.526, "dur": 62.944,
    "args": {
      "External id": 2463588,"kernel_hash": "cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/gh/cgh2wg2b6li2ipjwrhdr326ycb37fu4v576nwebfoyhwx5kppapn.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492049.087, "dur": 39.422,
    "args": {
      "External id": 2463589,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492108.404, "dur": 16.821,
    "args": {
      "External id": 2463590,"kernel_hash": "cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/ta/cta7vcczabyyuqarwmn4dcnazkxudhfwrvnick7ql5cmsd5lz4rz.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8189
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.27)", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492269.211, "dur": 34.162,
    "args": {
      "External id": 2463591,"Record function id": 0, "Ev Idx": 8190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492446.619, "dur": 315.327,
    "args": {
      "External id": 2463592,"Sequence number": 24740477, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8191
    }
  },
  {
    "ph": "s", "id": 171, "pid": 1336756, "tid": 1336756, "ts": 1514656492446.619,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492478.281, "dur": 7.924,
    "args": {
      "External id": 2463593,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492480.572, "dur": 5.340,
    "args": {
      "External id": 2463594,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492496.888, "dur": 11.063,
    "args": {
      "External id": 2463595,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492499.600, "dur": 7.797,
    "args": {
      "External id": 2463596,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492517.032, "dur": 4.754,
    "args": {
      "External id": 2463597,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492740.427, "dur": 7.676,
    "args": {
      "External id": 2463598,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492743.861, "dur": 3.870,
    "args": {
      "External id": 2463599,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492794.124, "dur": 134.696,
    "args": {
      "External id": 2463600,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492798.430, "dur": 18.398,
    "args": {
      "External id": 2463601,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8200
    }
  },
  {
    "ph": "s", "id": 170, "pid": 1336756, "tid": 1336756, "ts": 1514656492798.430,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492805.106, "dur": 9.822,
    "args": {
      "External id": 2463602,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492812.358, "dur": 2.198,
    "args": {
      "External id": 2463603,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492819.983, "dur": 108.513,
    "args": {
      "External id": 2463604,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492823.351, "dur": 6.594,
    "args": {
      "External id": 2463605,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492824.199, "dur": 5.568,
    "args": {
      "External id": 2463606,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8205
    }
  },
  {
    "ph": "s", "id": 169, "pid": 1336756, "tid": 1336756, "ts": 1514656492824.199,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492833.839, "dur": 84.605,
    "args": {
      "External id": 2463607,"Sequence number": 24740480, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8206
    }
  },
  {
    "ph": "s", "id": 168, "pid": 1336756, "tid": 1336756, "ts": 1514656492833.839,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492922.285, "dur": 5.173,
    "args": {
      "External id": 2463608,"Sequence number": 24740481, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8207
    }
  },
  {
    "ph": "s", "id": 167, "pid": 1336756, "tid": 1336756, "ts": 1514656492922.285,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492939.471, "dur": 106.663,
    "args": {
      "External id": 2463609,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492940.485, "dur": 8.766,
    "args": {
      "External id": 2463610,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8209
    }
  },
  {
    "ph": "s", "id": 166, "pid": 1336756, "tid": 1336756, "ts": 1514656492940.485,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492941.914, "dur": 6.145,
    "args": {
      "External id": 2463611,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492946.927, "dur": 0.930,
    "args": {
      "External id": 2463612,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492950.023, "dur": 95.810,
    "args": {
      "External id": 2463613,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492951.043, "dur": 4.522,
    "args": {
      "External id": 2463614,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492952.057, "dur": 3.134,
    "args": {
      "External id": 2463615,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8214
    }
  },
  {
    "ph": "s", "id": 165, "pid": 1336756, "tid": 1336756, "ts": 1514656492952.057,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656492956.272, "dur": 80.638,
    "args": {
      "External id": 2463616,"Sequence number": 24740484, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8215
    }
  },
  {
    "ph": "s", "id": 164, "pid": 1336756, "tid": 1336756, "ts": 1514656492956.272,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493040.263, "dur": 4.988,
    "args": {
      "External id": 2463617,"Sequence number": 24740485, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8216
    }
  },
  {
    "ph": "s", "id": 163, "pid": 1336756, "tid": 1336756, "ts": 1514656493040.263,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493056.753, "dur": 71.414,
    "args": {
      "External id": 2463618,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493057.784, "dur": 6.318,
    "args": {
      "External id": 2463619,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8218
    }
  },
  {
    "ph": "s", "id": 162, "pid": 1336756, "tid": 1336756, "ts": 1514656493057.784,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493059.706, "dur": 3.052,
    "args": {
      "External id": 2463620,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493061.367, "dur": 1.190,
    "args": {
      "External id": 2463621,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493066.459, "dur": 61.433,
    "args": {
      "External id": 2463622,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493067.885, "dur": 6.285,
    "args": {
      "External id": 2463623,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493068.844, "dur": 5.176,
    "args": {
      "External id": 2463624,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8223
    }
  },
  {
    "ph": "s", "id": 161, "pid": 1336756, "tid": 1336756, "ts": 1514656493068.844,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493074.853, "dur": 45.475,
    "args": {
      "External id": 2463625,"Sequence number": 24740488, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8224
    }
  },
  {
    "ph": "s", "id": 160, "pid": 1336756, "tid": 1336756, "ts": 1514656493074.853,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493121.713, "dur": 5.798,
    "args": {
      "External id": 2463626,"Sequence number": 24740489, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8225
    }
  },
  {
    "ph": "s", "id": 159, "pid": 1336756, "tid": 1336756, "ts": 1514656493121.713,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493152.888, "dur": 4.331,
    "args": {
      "External id": 2463627,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493153.647, "dur": 3.427,
    "args": {
      "External id": 2463628,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8227
    }
  },
  {
    "ph": "s", "id": 158, "pid": 1336756, "tid": 1336756, "ts": 1514656493153.647,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493163.772, "dur": 5.180,
    "args": {
      "External id": 2463629,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493165.055, "dur": 3.714,
    "args": {
      "External id": 2463630,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8229
    }
  },
  {
    "ph": "s", "id": 157, "pid": 1336756, "tid": 1336756, "ts": 1514656493165.055,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493175.498, "dur": 3.123,
    "args": {
      "External id": 2463631,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493176.513, "dur": 1.965,
    "args": {
      "External id": 2463632,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8231
    }
  },
  {
    "ph": "s", "id": 156, "pid": 1336756, "tid": 1336756, "ts": 1514656493176.513,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493214.254, "dur": 184.260,
    "args": {
      "External id": 2463633,"Sequence number": 24740493, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8232
    }
  },
  {
    "ph": "s", "id": 155, "pid": 1336756, "tid": 1336756, "ts": 1514656493214.254,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493237.801, "dur": 9.402,
    "args": {
      "External id": 2463634,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493240.928, "dur": 5.728,
    "args": {
      "External id": 2463635,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493411.395, "dur": 110.097,
    "args": {
      "External id": 2463636,"Sequence number": 24740494, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8235
    }
  },
  {
    "ph": "s", "id": 154, "pid": 1336756, "tid": 1336756, "ts": 1514656493411.395,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493426.083, "dur": 6.669,
    "args": {
      "External id": 2463637,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493428.174, "dur": 4.140,
    "args": {
      "External id": 2463638,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493570.916, "dur": 195.969,
    "args": {
      "External id": 2463639,"Sequence number": 24740495, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8238
    }
  },
  {
    "ph": "s", "id": 153, "pid": 1336756, "tid": 1336756, "ts": 1514656493570.916,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493603.163, "dur": 130.341,
    "args": {
      "External id": 2463640,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493654.063, "dur": 8.680,
    "args": {
      "External id": 2463641,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493656.485, "dur": 5.597,
    "args": {
      "External id": 2463642,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493665.263, "dur": 3.831,
    "args": {
      "External id": 2463643,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493672.460, "dur": 1.495,
    "args": {
      "External id": 2463644,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493676.242, "dur": 3.116,
    "args": {
      "External id": 2463645,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493749.666, "dur": 6.249,
    "args": {
      "External id": 2463646,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493771.873, "dur": 5.600,
    "args": {
      "External id": 2463647,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493773.256, "dur": 4.048,
    "args": {
      "External id": 2463648,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8247
    }
  },
  {
    "ph": "s", "id": 152, "pid": 1336756, "tid": 1336756, "ts": 1514656493773.256,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493790.475, "dur": 115.404,
    "args": {
      "External id": 2463649,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493793.815, "dur": 8.346,
    "args": {
      "External id": 2463650,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8249
    }
  },
  {
    "ph": "s", "id": 151, "pid": 1336756, "tid": 1336756, "ts": 1514656493793.815,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493796.797, "dur": 4.302,
    "args": {
      "External id": 2463651,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493799.066, "dur": 1.773,
    "args": {
      "External id": 2463652,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493803.375, "dur": 102.134,
    "args": {
      "External id": 2463653,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493804.884, "dur": 7.750,
    "args": {
      "External id": 2463654,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493807.591, "dur": 4.904,
    "args": {
      "External id": 2463655,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8254
    }
  },
  {
    "ph": "s", "id": 150, "pid": 1336756, "tid": 1336756, "ts": 1514656493807.591,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493813.597, "dur": 85.421,
    "args": {
      "External id": 2463656,"Sequence number": 24740499, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8255
    }
  },
  {
    "ph": "s", "id": 149, "pid": 1336756, "tid": 1336756, "ts": 1514656493813.597,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493901.389, "dur": 3.410,
    "args": {
      "External id": 2463657,"Sequence number": 24740500, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8256
    }
  },
  {
    "ph": "s", "id": 148, "pid": 1336756, "tid": 1336756, "ts": 1514656493901.389,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493940.594, "dur": 285.546,
    "args": {
      "External id": 2463658,"Sequence number": 24740501, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8257
    }
  },
  {
    "ph": "s", "id": 147, "pid": 1336756, "tid": 1336756, "ts": 1514656493940.594,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493961.101, "dur": 6.955,
    "args": {
      "External id": 2463659,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493965.712, "dur": 2.151,
    "args": {
      "External id": 2463660,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493974.386, "dur": 3.703,
    "args": {
      "External id": 2463661,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493975.698, "dur": 2.279,
    "args": {
      "External id": 2463662,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656493976.604, "dur": 1.269,
    "args": {
      "External id": 2463663,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494024.461, "dur": 10.806,
    "args": {
      "External id": 2463664,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494028.818, "dur": 5.918,
    "args": {
      "External id": 2463665,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494042.162, "dur": 3.747,
    "args": {
      "External id": 2463666,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494051.353, "dur": 2.604,
    "args": {
      "External id": 2463667,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494199.801, "dur": 4.127,
    "args": {
      "External id": 2463668,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494200.741, "dur": 2.973,
    "args": {
      "External id": 2463669,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494206.769, "dur": 2.402,
    "args": {
      "External id": 2463670,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494207.473, "dur": 1.577,
    "args": {
      "External id": 2463671,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494250.156, "dur": 97.971,
    "args": {
      "External id": 2463672,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494251.443, "dur": 8.374,
    "args": {
      "External id": 2463673,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8272
    }
  },
  {
    "ph": "s", "id": 146, "pid": 1336756, "tid": 1336756, "ts": 1514656494251.443,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494254.032, "dur": 4.554,
    "args": {
      "External id": 2463674,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494256.245, "dur": 1.954,
    "args": {
      "External id": 2463675,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494260.781, "dur": 86.953,
    "args": {
      "External id": 2463676,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494264.387, "dur": 3.731,
    "args": {
      "External id": 2463677,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494265.159, "dur": 2.846,
    "args": {
      "External id": 2463678,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8277
    }
  },
  {
    "ph": "s", "id": 145, "pid": 1336756, "tid": 1336756, "ts": 1514656494265.159,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494269.036, "dur": 70.766,
    "args": {
      "External id": 2463679,"Sequence number": 24740504, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8278
    }
  },
  {
    "ph": "s", "id": 144, "pid": 1336756, "tid": 1336756, "ts": 1514656494269.036,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494342.028, "dur": 5.092,
    "args": {
      "External id": 2463680,"Sequence number": 24740505, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8279
    }
  },
  {
    "ph": "s", "id": 143, "pid": 1336756, "tid": 1336756, "ts": 1514656494342.028,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494356.892, "dur": 70.000,
    "args": {
      "External id": 2463681,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494359.640, "dur": 5.564,
    "args": {
      "External id": 2463682,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8281
    }
  },
  {
    "ph": "s", "id": 142, "pid": 1336756, "tid": 1336756, "ts": 1514656494359.640,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494361.216, "dur": 2.904,
    "args": {
      "External id": 2463683,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494363.063, "dur": 0.890,
    "args": {
      "External id": 2463684,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494365.891, "dur": 60.815,
    "args": {
      "External id": 2463685,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494366.886, "dur": 7.378,
    "args": {
      "External id": 2463686,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494369.384, "dur": 4.756,
    "args": {
      "External id": 2463687,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8286
    }
  },
  {
    "ph": "s", "id": 141, "pid": 1336756, "tid": 1336756, "ts": 1514656494369.384,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494375.010, "dur": 47.319,
    "args": {
      "External id": 2463688,"Sequence number": 24740508, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8287
    }
  },
  {
    "ph": "s", "id": 140, "pid": 1336756, "tid": 1336756, "ts": 1514656494375.010,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494423.947, "dur": 2.431,
    "args": {
      "External id": 2463689,"Sequence number": 24740509, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8288
    }
  },
  {
    "ph": "s", "id": 139, "pid": 1336756, "tid": 1336756, "ts": 1514656494423.947,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494452.041, "dur": 189.798,
    "args": {
      "External id": 2463690,"Sequence number": 24740510, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8289
    }
  },
  {
    "ph": "s", "id": 138, "pid": 1336756, "tid": 1336756, "ts": 1514656494452.041,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494500.053, "dur": 4.180,
    "args": {
      "External id": 2463691,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494560.107, "dur": 66.367,
    "args": {
      "External id": 2463692,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494560.910, "dur": 7.351,
    "args": {
      "External id": 2463693,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494562.312, "dur": 5.027,
    "args": {
      "External id": 2463694,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494565.883, "dur": 1.143,
    "args": {
      "External id": 2463695,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494569.079, "dur": 56.961,
    "args": {
      "External id": 2463696,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494572.362, "dur": 3.388,
    "args": {
      "External id": 2463697,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494573.407, "dur": 2.232,
    "args": {
      "External id": 2463698,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494576.492, "dur": 45.745,
    "args": {
      "External id": 2463699,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494624.327, "dur": 1.091,
    "args": {
      "External id": 2463700,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494653.356, "dur": 29.171,
    "args": {
      "External id": 2463701,"Sequence number": 24740511, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8300
    }
  },
  {
    "ph": "s", "id": 137, "pid": 1336756, "tid": 1336756, "ts": 1514656494653.356,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494719.467, "dur": 182.257,
    "args": {
      "External id": 2463702,"Sequence number": 24740512, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8301
    }
  },
  {
    "ph": "s", "id": 136, "pid": 1336756, "tid": 1336756, "ts": 1514656494719.467,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494739.646, "dur": 5.298,
    "args": {
      "External id": 2463703,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494742.475, "dur": 2.176,
    "args": {
      "External id": 2463704,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494752.540, "dur": 6.952,
    "args": {
      "External id": 2463705,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494755.193, "dur": 3.941,
    "args": {
      "External id": 2463706,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494765.243, "dur": 3.764,
    "args": {
      "External id": 2463707,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494887.474, "dur": 2.921,
    "args": {
      "External id": 2463708,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494888.440, "dur": 1.742,
    "args": {
      "External id": 2463709,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494922.428, "dur": 117.856,
    "args": {
      "External id": 2463710,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494923.446, "dur": 7.879,
    "args": {
      "External id": 2463711,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8310
    }
  },
  {
    "ph": "s", "id": 135, "pid": 1336756, "tid": 1336756, "ts": 1514656494923.446,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494926.894, "dur": 3.320,
    "args": {
      "External id": 2463712,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494928.706, "dur": 1.239,
    "args": {
      "External id": 2463713,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494932.328, "dur": 107.621,
    "args": {
      "External id": 2463714,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494933.737, "dur": 5.381,
    "args": {
      "External id": 2463715,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494936.136, "dur": 2.820,
    "args": {
      "External id": 2463716,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8315
    }
  },
  {
    "ph": "s", "id": 134, "pid": 1336756, "tid": 1336756, "ts": 1514656494936.136,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656494939.825, "dur": 90.745,
    "args": {
      "External id": 2463717,"Sequence number": 24740515, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8316
    }
  },
  {
    "ph": "s", "id": 133, "pid": 1336756, "tid": 1336756, "ts": 1514656494939.825,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495034.064, "dur": 4.977,
    "args": {
      "External id": 2463718,"Sequence number": 24740516, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8317
    }
  },
  {
    "ph": "s", "id": 132, "pid": 1336756, "tid": 1336756, "ts": 1514656495034.064,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495051.566, "dur": 72.487,
    "args": {
      "External id": 2463719,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495052.599, "dur": 9.140,
    "args": {
      "External id": 2463720,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8319
    }
  },
  {
    "ph": "s", "id": 131, "pid": 1336756, "tid": 1336756, "ts": 1514656495052.599,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495057.667, "dur": 2.878,
    "args": {
      "External id": 2463721,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495059.346, "dur": 0.934,
    "args": {
      "External id": 2463722,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495062.791, "dur": 61.015,
    "args": {
      "External id": 2463723,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495063.820, "dur": 5.096,
    "args": {
      "External id": 2463724,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495064.759, "dur": 3.990,
    "args": {
      "External id": 2463725,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8324
    }
  },
  {
    "ph": "s", "id": 130, "pid": 1336756, "tid": 1336756, "ts": 1514656495064.759,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495069.594, "dur": 50.202,
    "args": {
      "External id": 2463726,"Sequence number": 24740519, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8325
    }
  },
  {
    "ph": "s", "id": 129, "pid": 1336756, "tid": 1336756, "ts": 1514656495069.594,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495121.388, "dur": 2.069,
    "args": {
      "External id": 2463727,"Sequence number": 24740520, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8326
    }
  },
  {
    "ph": "s", "id": 128, "pid": 1336756, "tid": 1336756, "ts": 1514656495121.388,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495131.781, "dur": 61.667,
    "args": {
      "External id": 2463728,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495132.429, "dur": 7.316,
    "args": {
      "External id": 2463729,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8328
    }
  },
  {
    "ph": "s", "id": 127, "pid": 1336756, "tid": 1336756, "ts": 1514656495132.429,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495134.436, "dur": 4.029,
    "args": {
      "External id": 2463730,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495137.706, "dur": 0.601,
    "args": {
      "External id": 2463731,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495140.263, "dur": 52.909,
    "args": {
      "External id": 2463732,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495141.307, "dur": 5.589,
    "args": {
      "External id": 2463733,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495142.439, "dur": 4.329,
    "args": {
      "External id": 2463734,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8333
    }
  },
  {
    "ph": "s", "id": 126, "pid": 1336756, "tid": 1336756, "ts": 1514656495142.439,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495149.193, "dur": 39.755,
    "args": {
      "External id": 2463735,"Sequence number": 24740523, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8334
    }
  },
  {
    "ph": "s", "id": 125, "pid": 1336756, "tid": 1336756, "ts": 1514656495149.193,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495190.459, "dur": 2.447,
    "args": {
      "External id": 2463736,"Sequence number": 24740524, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8335
    }
  },
  {
    "ph": "s", "id": 124, "pid": 1336756, "tid": 1336756, "ts": 1514656495190.459,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495208.999, "dur": 4.502,
    "args": {
      "External id": 2463737,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495209.954, "dur": 3.390,
    "args": {
      "External id": 2463738,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8337
    }
  },
  {
    "ph": "s", "id": 123, "pid": 1336756, "tid": 1336756, "ts": 1514656495209.954,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495220.141, "dur": 9.490,
    "args": {
      "External id": 2463739,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495223.257, "dur": 6.207,
    "args": {
      "External id": 2463740,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8339
    }
  },
  {
    "ph": "s", "id": 122, "pid": 1336756, "tid": 1336756, "ts": 1514656495223.257,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495233.719, "dur": 3.219,
    "args": {
      "External id": 2463741,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495235.055, "dur": 1.735,
    "args": {
      "External id": 2463742,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8341
    }
  },
  {
    "ph": "s", "id": 121, "pid": 1336756, "tid": 1336756, "ts": 1514656495235.055,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495265.359, "dur": 185.487,
    "args": {
      "External id": 2463743,"Sequence number": 24740528, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8342
    }
  },
  {
    "ph": "s", "id": 120, "pid": 1336756, "tid": 1336756, "ts": 1514656495265.359,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495284.312, "dur": 35.820,
    "args": {
      "External id": 2463744,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495314.496, "dur": 5.157,
    "args": {
      "External id": 2463745,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495465.342, "dur": 126.557,
    "args": {
      "External id": 2463746,"Sequence number": 24740529, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8345
    }
  },
  {
    "ph": "s", "id": 119, "pid": 1336756, "tid": 1336756, "ts": 1514656495465.342,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495477.662, "dur": 6.251,
    "args": {
      "External id": 2463747,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495479.705, "dur": 3.831,
    "args": {
      "External id": 2463748,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495621.550, "dur": 172.105,
    "args": {
      "External id": 2463749,"Sequence number": 24740530, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8348
    }
  },
  {
    "ph": "s", "id": 118, "pid": 1336756, "tid": 1336756, "ts": 1514656495621.550,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495651.347, "dur": 117.897,
    "args": {
      "External id": 2463750,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495698.138, "dur": 7.720,
    "args": {
      "External id": 2463751,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495700.494, "dur": 4.913,
    "args": {
      "External id": 2463752,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495708.147, "dur": 3.967,
    "args": {
      "External id": 2463753,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495713.155, "dur": 1.590,
    "args": {
      "External id": 2463754,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495717.009, "dur": 3.058,
    "args": {
      "External id": 2463755,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495779.926, "dur": 4.278,
    "args": {
      "External id": 2463756,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495798.229, "dur": 7.287,
    "args": {
      "External id": 2463757,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495799.548, "dur": 5.794,
    "args": {
      "External id": 2463758,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8357
    }
  },
  {
    "ph": "s", "id": 117, "pid": 1336756, "tid": 1336756, "ts": 1514656495799.548,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495820.745, "dur": 107.439,
    "args": {
      "External id": 2463759,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495821.786, "dur": 9.329,
    "args": {
      "External id": 2463760,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8359
    }
  },
  {
    "ph": "s", "id": 116, "pid": 1336756, "tid": 1336756, "ts": 1514656495821.786,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495824.180, "dur": 5.666,
    "args": {
      "External id": 2463761,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495828.021, "dur": 1.584,
    "args": {
      "External id": 2463762,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495832.241, "dur": 95.578,
    "args": {
      "External id": 2463763,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495835.666, "dur": 4.565,
    "args": {
      "External id": 2463764,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495836.075, "dur": 4.010,
    "args": {
      "External id": 2463765,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8364
    }
  },
  {
    "ph": "s", "id": 115, "pid": 1336756, "tid": 1336756, "ts": 1514656495836.075,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495841.227, "dur": 78.347,
    "args": {
      "External id": 2463766,"Sequence number": 24740534, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8365
    }
  },
  {
    "ph": "s", "id": 114, "pid": 1336756, "tid": 1336756, "ts": 1514656495841.227,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495921.597, "dur": 5.530,
    "args": {
      "External id": 2463767,"Sequence number": 24740535, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8366
    }
  },
  {
    "ph": "s", "id": 113, "pid": 1336756, "tid": 1336756, "ts": 1514656495921.597,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495960.275, "dur": 260.258,
    "args": {
      "External id": 2463768,"Sequence number": 24740536, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8367
    }
  },
  {
    "ph": "s", "id": 112, "pid": 1336756, "tid": 1336756, "ts": 1514656495960.275,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495981.501, "dur": 2.434,
    "args": {
      "External id": 2463769,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656495982.167, "dur": 1.614,
    "args": {
      "External id": 2463770,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496025.705, "dur": 6.287,
    "args": {
      "External id": 2463771,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496026.888, "dur": 4.994,
    "args": {
      "External id": 2463772,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496029.450, "dur": 2.130,
    "args": {
      "External id": 2463773,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496039.867, "dur": 8.087,
    "args": {
      "External id": 2463774,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496041.971, "dur": 5.671,
    "args": {
      "External id": 2463775,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496054.277, "dur": 3.423,
    "args": {
      "External id": 2463776,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496061.434, "dur": 3.952,
    "args": {
      "External id": 2463777,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496197.163, "dur": 3.212,
    "args": {
      "External id": 2463778,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496198.111, "dur": 2.031,
    "args": {
      "External id": 2463779,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496204.130, "dur": 2.349,
    "args": {
      "External id": 2463780,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496204.957, "dur": 1.405,
    "args": {
      "External id": 2463781,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496241.700, "dur": 97.526,
    "args": {
      "External id": 2463782,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496242.891, "dur": 8.254,
    "args": {
      "External id": 2463783,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8382
    }
  },
  {
    "ph": "s", "id": 111, "pid": 1336756, "tid": 1336756, "ts": 1514656496242.891,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496245.873, "dur": 3.777,
    "args": {
      "External id": 2463784,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496247.954, "dur": 1.353,
    "args": {
      "External id": 2463785,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496254.613, "dur": 84.284,
    "args": {
      "External id": 2463786,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496256.194, "dur": 3.085,
    "args": {
      "External id": 2463787,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496256.880, "dur": 2.231,
    "args": {
      "External id": 2463788,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8387
    }
  },
  {
    "ph": "s", "id": 110, "pid": 1336756, "tid": 1336756, "ts": 1514656496256.880,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496259.959, "dur": 71.463,
    "args": {
      "External id": 2463789,"Sequence number": 24740539, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8388
    }
  },
  {
    "ph": "s", "id": 109, "pid": 1336756, "tid": 1336756, "ts": 1514656496259.959,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496333.567, "dur": 4.656,
    "args": {
      "External id": 2463790,"Sequence number": 24740540, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8389
    }
  },
  {
    "ph": "s", "id": 108, "pid": 1336756, "tid": 1336756, "ts": 1514656496333.567,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496348.138, "dur": 68.810,
    "args": {
      "External id": 2463791,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496348.934, "dur": 5.983,
    "args": {
      "External id": 2463792,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8391
    }
  },
  {
    "ph": "s", "id": 107, "pid": 1336756, "tid": 1336756, "ts": 1514656496348.934,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496350.843, "dur": 2.775,
    "args": {
      "External id": 2463793,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496352.535, "dur": 0.925,
    "args": {
      "External id": 2463794,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496355.681, "dur": 60.972,
    "args": {
      "External id": 2463795,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496358.875, "dur": 7.146,
    "args": {
      "External id": 2463796,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496359.392, "dur": 6.462,
    "args": {
      "External id": 2463797,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8396
    }
  },
  {
    "ph": "s", "id": 106, "pid": 1336756, "tid": 1336756, "ts": 1514656496359.392,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496366.574, "dur": 44.524,
    "args": {
      "External id": 2463798,"Sequence number": 24740543, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8397
    }
  },
  {
    "ph": "s", "id": 105, "pid": 1336756, "tid": 1336756, "ts": 1514656496366.574,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496412.418, "dur": 3.888,
    "args": {
      "External id": 2463799,"Sequence number": 24740544, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8398
    }
  },
  {
    "ph": "s", "id": 104, "pid": 1336756, "tid": 1336756, "ts": 1514656496412.418,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496441.010, "dur": 163.633,
    "args": {
      "External id": 2463800,"Sequence number": 24740545, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8399
    }
  },
  {
    "ph": "s", "id": 103, "pid": 1336756, "tid": 1336756, "ts": 1514656496441.010,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496474.817, "dur": 3.995,
    "args": {
      "External id": 2463801,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496507.888, "dur": 81.994,
    "args": {
      "External id": 2463802,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496508.612, "dur": 4.974,
    "args": {
      "External id": 2463803,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496509.948, "dur": 2.868,
    "args": {
      "External id": 2463804,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496511.742, "dur": 0.769,
    "args": {
      "External id": 2463805,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496516.334, "dur": 73.254,
    "args": {
      "External id": 2463806,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496517.331, "dur": 2.760,
    "args": {
      "External id": 2463807,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496518.581, "dur": 1.394,
    "args": {
      "External id": 2463808,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496520.743, "dur": 63.668,
    "args": {
      "External id": 2463809,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496587.439, "dur": 1.441,
    "args": {
      "External id": 2463810,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496613.771, "dur": 23.937,
    "args": {
      "External id": 2463811,"Sequence number": 24740546, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8410
    }
  },
  {
    "ph": "s", "id": 102, "pid": 1336756, "tid": 1336756, "ts": 1514656496613.771,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496675.070, "dur": 176.125,
    "args": {
      "External id": 2463812,"Sequence number": 24740547, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8411
    }
  },
  {
    "ph": "s", "id": 101, "pid": 1336756, "tid": 1336756, "ts": 1514656496675.070,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496694.898, "dur": 2.942,
    "args": {
      "External id": 2463813,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496695.662, "dur": 2.039,
    "args": {
      "External id": 2463814,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496705.799, "dur": 6.592,
    "args": {
      "External id": 2463815,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496708.137, "dur": 3.861,
    "args": {
      "External id": 2463816,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496718.656, "dur": 3.682,
    "args": {
      "External id": 2463817,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496836.370, "dur": 2.885,
    "args": {
      "External id": 2463818,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496837.516, "dur": 1.549,
    "args": {
      "External id": 2463819,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496870.917, "dur": 81.553,
    "args": {
      "External id": 2463820,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496871.806, "dur": 6.786,
    "args": {
      "External id": 2463821,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8420
    }
  },
  {
    "ph": "s", "id": 100, "pid": 1336756, "tid": 1336756, "ts": 1514656496871.806,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496874.218, "dur": 3.200,
    "args": {
      "External id": 2463822,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496875.874, "dur": 1.307,
    "args": {
      "External id": 2463823,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496879.670, "dur": 72.525,
    "args": {
      "External id": 2463824,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496882.850, "dur": 3.563,
    "args": {
      "External id": 2463825,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496883.482, "dur": 2.806,
    "args": {
      "External id": 2463826,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8425
    }
  },
  {
    "ph": "s", "id": 99, "pid": 1336756, "tid": 1336756, "ts": 1514656496883.482,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496887.006, "dur": 57.321,
    "args": {
      "External id": 2463827,"Sequence number": 24740550, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8426
    }
  },
  {
    "ph": "s", "id": 98, "pid": 1336756, "tid": 1336756, "ts": 1514656496887.006,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496946.060, "dur": 5.626,
    "args": {
      "External id": 2463828,"Sequence number": 24740551, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8427
    }
  },
  {
    "ph": "s", "id": 97, "pid": 1336756, "tid": 1336756, "ts": 1514656496946.060,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496961.025, "dur": 105.153,
    "args": {
      "External id": 2463829,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496963.439, "dur": 5.864,
    "args": {
      "External id": 2463830,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8429
    }
  },
  {
    "ph": "s", "id": 96, "pid": 1336756, "tid": 1336756, "ts": 1514656496963.439,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496965.477, "dur": 2.683,
    "args": {
      "External id": 2463831,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496967.072, "dur": 0.879,
    "args": {
      "External id": 2463832,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496970.069, "dur": 95.865,
    "args": {
      "External id": 2463833,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496970.984, "dur": 4.944,
    "args": {
      "External id": 2463834,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496973.720, "dur": 2.070,
    "args": {
      "External id": 2463835,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8434
    }
  },
  {
    "ph": "s", "id": 95, "pid": 1336756, "tid": 1336756, "ts": 1514656496973.720,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656496976.710, "dur": 83.266,
    "args": {
      "External id": 2463836,"Sequence number": 24740554, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8435
    }
  },
  {
    "ph": "s", "id": 94, "pid": 1336756, "tid": 1336756, "ts": 1514656496976.710,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497063.035, "dur": 2.458,
    "args": {
      "External id": 2463837,"Sequence number": 24740555, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8436
    }
  },
  {
    "ph": "s", "id": 93, "pid": 1336756, "tid": 1336756, "ts": 1514656497063.035,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497076.225, "dur": 64.599,
    "args": {
      "External id": 2463838,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497076.960, "dur": 8.449,
    "args": {
      "External id": 2463839,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8438
    }
  },
  {
    "ph": "s", "id": 92, "pid": 1336756, "tid": 1336756, "ts": 1514656497076.960,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497080.203, "dur": 4.042,
    "args": {
      "External id": 2463840,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497081.727, "dur": 2.313,
    "args": {
      "External id": 2463841,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497086.278, "dur": 54.332,
    "args": {
      "External id": 2463842,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497087.780, "dur": 7.486,
    "args": {
      "External id": 2463843,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497090.498, "dur": 4.635,
    "args": {
      "External id": 2463844,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8443
    }
  },
  {
    "ph": "s", "id": 91, "pid": 1336756, "tid": 1336756, "ts": 1514656497090.498,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497095.876, "dur": 39.571,
    "args": {
      "External id": 2463845,"Sequence number": 24740558, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8444
    }
  },
  {
    "ph": "s", "id": 90, "pid": 1336756, "tid": 1336756, "ts": 1514656497095.876,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497136.764, "dur": 3.518,
    "args": {
      "External id": 2463846,"Sequence number": 24740559, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8445
    }
  },
  {
    "ph": "s", "id": 89, "pid": 1336756, "tid": 1336756, "ts": 1514656497136.764,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497156.814, "dur": 4.879,
    "args": {
      "External id": 2463847,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497157.633, "dur": 3.897,
    "args": {
      "External id": 2463848,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8447
    }
  },
  {
    "ph": "s", "id": 88, "pid": 1336756, "tid": 1336756, "ts": 1514656497157.633,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497168.184, "dur": 3.013,
    "args": {
      "External id": 2463849,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497169.241, "dur": 1.803,
    "args": {
      "External id": 2463850,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8449
    }
  },
  {
    "ph": "s", "id": 87, "pid": 1336756, "tid": 1336756, "ts": 1514656497169.241,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497175.753, "dur": 3.254,
    "args": {
      "External id": 2463851,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497176.547, "dur": 2.314,
    "args": {
      "External id": 2463852,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8451
    }
  },
  {
    "ph": "s", "id": 86, "pid": 1336756, "tid": 1336756, "ts": 1514656497176.547,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497207.300, "dur": 153.017,
    "args": {
      "External id": 2463853,"Sequence number": 24740563, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8452
    }
  },
  {
    "ph": "s", "id": 85, "pid": 1336756, "tid": 1336756, "ts": 1514656497207.300,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497227.491, "dur": 9.162,
    "args": {
      "External id": 2463854,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497229.748, "dur": 6.435,
    "args": {
      "External id": 2463855,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497371.836, "dur": 102.251,
    "args": {
      "External id": 2463856,"Sequence number": 24740564, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8455
    }
  },
  {
    "ph": "s", "id": 84, "pid": 1336756, "tid": 1336756, "ts": 1514656497371.836,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497385.270, "dur": 6.416,
    "args": {
      "External id": 2463857,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497387.383, "dur": 3.927,
    "args": {
      "External id": 2463858,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497500.456, "dur": 189.021,
    "args": {
      "External id": 2463859,"Sequence number": 24740565, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8458
    }
  },
  {
    "ph": "s", "id": 83, "pid": 1336756, "tid": 1336756, "ts": 1514656497500.456,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497542.533, "dur": 122.017,
    "args": {
      "External id": 2463860,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497590.867, "dur": 8.269,
    "args": {
      "External id": 2463861,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497592.987, "dur": 5.343,
    "args": {
      "External id": 2463862,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497601.835, "dur": 3.785,
    "args": {
      "External id": 2463863,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497607.071, "dur": 1.420,
    "args": {
      "External id": 2463864,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497610.840, "dur": 3.510,
    "args": {
      "External id": 2463865,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497675.409, "dur": 4.620,
    "args": {
      "External id": 2463866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497694.070, "dur": 5.181,
    "args": {
      "External id": 2463867,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497695.456, "dur": 3.654,
    "args": {
      "External id": 2463868,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8467
    }
  },
  {
    "ph": "s", "id": 82, "pid": 1336756, "tid": 1336756, "ts": 1514656497695.456,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497711.176, "dur": 101.553,
    "args": {
      "External id": 2463869,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497712.454, "dur": 12.062,
    "args": {
      "External id": 2463870,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8469
    }
  },
  {
    "ph": "s", "id": 81, "pid": 1336756, "tid": 1336756, "ts": 1514656497712.454,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497718.186, "dur": 5.215,
    "args": {
      "External id": 2463871,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497721.649, "dur": 1.526,
    "args": {
      "External id": 2463872,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497725.765, "dur": 86.607,
    "args": {
      "External id": 2463873,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497727.606, "dur": 3.199,
    "args": {
      "External id": 2463874,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497728.505, "dur": 2.200,
    "args": {
      "External id": 2463875,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8474
    }
  },
  {
    "ph": "s", "id": 80, "pid": 1336756, "tid": 1336756, "ts": 1514656497728.505,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497731.899, "dur": 72.974,
    "args": {
      "External id": 2463876,"Sequence number": 24740569, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8475
    }
  },
  {
    "ph": "s", "id": 79, "pid": 1336756, "tid": 1336756, "ts": 1514656497731.899,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497806.718, "dur": 4.986,
    "args": {
      "External id": 2463877,"Sequence number": 24740570, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8476
    }
  },
  {
    "ph": "s", "id": 78, "pid": 1336756, "tid": 1336756, "ts": 1514656497806.718,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497845.169, "dur": 242.409,
    "args": {
      "External id": 2463878,"Sequence number": 24740571, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8477
    }
  },
  {
    "ph": "s", "id": 77, "pid": 1336756, "tid": 1336756, "ts": 1514656497845.169,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497862.292, "dur": 3.316,
    "args": {
      "External id": 2463879,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497863.676, "dur": 1.788,
    "args": {
      "External id": 2463880,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497869.088, "dur": 4.988,
    "args": {
      "External id": 2463881,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497871.982, "dur": 1.972,
    "args": {
      "External id": 2463882,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497872.789, "dur": 1.040,
    "args": {
      "External id": 2463883,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497881.191, "dur": 5.896,
    "args": {
      "External id": 2463884,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497882.687, "dur": 4.121,
    "args": {
      "External id": 2463885,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497893.328, "dur": 3.025,
    "args": {
      "External id": 2463886,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656497900.067, "dur": 3.532,
    "args": {
      "External id": 2463887,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498063.087, "dur": 4.151,
    "args": {
      "External id": 2463888,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498064.071, "dur": 2.854,
    "args": {
      "External id": 2463889,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498070.180, "dur": 2.301,
    "args": {
      "External id": 2463890,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498071.187, "dur": 1.166,
    "args": {
      "External id": 2463891,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498107.230, "dur": 92.953,
    "args": {
      "External id": 2463892,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498108.135, "dur": 9.428,
    "args": {
      "External id": 2463893,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8492
    }
  },
  {
    "ph": "s", "id": 76, "pid": 1336756, "tid": 1336756, "ts": 1514656498108.135,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498110.891, "dur": 5.359,
    "args": {
      "External id": 2463894,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498114.761, "dur": 1.284,
    "args": {
      "External id": 2463895,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498118.777, "dur": 81.129,
    "args": {
      "External id": 2463896,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498120.292, "dur": 3.816,
    "args": {
      "External id": 2463897,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498121.242, "dur": 2.752,
    "args": {
      "External id": 2463898,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8497
    }
  },
  {
    "ph": "s", "id": 75, "pid": 1336756, "tid": 1336756, "ts": 1514656498121.242,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498126.442, "dur": 65.867,
    "args": {
      "External id": 2463899,"Sequence number": 24740574, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8498
    }
  },
  {
    "ph": "s", "id": 74, "pid": 1336756, "tid": 1336756, "ts": 1514656498126.442,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498194.235, "dur": 5.171,
    "args": {
      "External id": 2463900,"Sequence number": 24740575, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8499
    }
  },
  {
    "ph": "s", "id": 73, "pid": 1336756, "tid": 1336756, "ts": 1514656498194.235,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498208.083, "dur": 84.188,
    "args": {
      "External id": 2463901,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498208.726, "dur": 22.715,
    "args": {
      "External id": 2463902,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8501
    }
  },
  {
    "ph": "s", "id": 72, "pid": 1336756, "tid": 1336756, "ts": 1514656498208.726,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498210.669, "dur": 13.410,
    "args": {
      "External id": 2463903,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498221.342, "dur": 2.496,
    "args": {
      "External id": 2463904,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498232.106, "dur": 59.835,
    "args": {
      "External id": 2463905,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498233.124, "dur": 6.113,
    "args": {
      "External id": 2463906,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498234.441, "dur": 4.667,
    "args": {
      "External id": 2463907,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8506
    }
  },
  {
    "ph": "s", "id": 71, "pid": 1336756, "tid": 1336756, "ts": 1514656498234.441,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498239.842, "dur": 46.381,
    "args": {
      "External id": 2463908,"Sequence number": 24740578, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8507
    }
  },
  {
    "ph": "s", "id": 70, "pid": 1336756, "tid": 1336756, "ts": 1514656498239.842,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498287.666, "dur": 3.909,
    "args": {
      "External id": 2463909,"Sequence number": 24740579, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8508
    }
  },
  {
    "ph": "s", "id": 69, "pid": 1336756, "tid": 1336756, "ts": 1514656498287.666,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498313.380, "dur": 133.944,
    "args": {
      "External id": 2463910,"Sequence number": 24740580, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8509
    }
  },
  {
    "ph": "s", "id": 68, "pid": 1336756, "tid": 1336756, "ts": 1514656498313.380,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498347.557, "dur": 4.644,
    "args": {
      "External id": 2463911,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498381.983, "dur": 53.910,
    "args": {
      "External id": 2463912,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498382.716, "dur": 4.572,
    "args": {
      "External id": 2463913,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498383.644, "dur": 2.817,
    "args": {
      "External id": 2463914,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498385.401, "dur": 0.891,
    "args": {
      "External id": 2463915,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498388.084, "dur": 47.198,
    "args": {
      "External id": 2463916,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498389.093, "dur": 2.169,
    "args": {
      "External id": 2463917,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498389.775, "dur": 1.370,
    "args": {
      "External id": 2463918,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498391.772, "dur": 40.068,
    "args": {
      "External id": 2463919,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498433.719, "dur": 1.045,
    "args": {
      "External id": 2463920,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498454.926, "dur": 23.139,
    "args": {
      "External id": 2463921,"Sequence number": 24740581, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8520
    }
  },
  {
    "ph": "s", "id": 67, "pid": 1336756, "tid": 1336756, "ts": 1514656498454.926,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498512.430, "dur": 214.846,
    "args": {
      "External id": 2463922,"Sequence number": 24740582, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8521
    }
  },
  {
    "ph": "s", "id": 66, "pid": 1336756, "tid": 1336756, "ts": 1514656498512.430,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498544.782, "dur": 4.369,
    "args": {
      "External id": 2463923,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498545.978, "dur": 2.838,
    "args": {
      "External id": 2463924,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498557.668, "dur": 8.046,
    "args": {
      "External id": 2463925,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498560.294, "dur": 5.066,
    "args": {
      "External id": 2463926,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498572.419, "dur": 3.881,
    "args": {
      "External id": 2463927,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498700.094, "dur": 16.613,
    "args": {
      "External id": 2463928,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498703.948, "dur": 12.502,
    "args": {
      "External id": 2463929,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498749.224, "dur": 93.403,
    "args": {
      "External id": 2463930,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498750.091, "dur": 9.540,
    "args": {
      "External id": 2463931,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8530
    }
  },
  {
    "ph": "s", "id": 65, "pid": 1336756, "tid": 1336756, "ts": 1514656498750.091,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498752.504, "dur": 5.789,
    "args": {
      "External id": 2463932,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498756.383, "dur": 1.671,
    "args": {
      "External id": 2463933,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498760.819, "dur": 81.434,
    "args": {
      "External id": 2463934,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498762.184, "dur": 3.997,
    "args": {
      "External id": 2463935,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498763.274, "dur": 2.763,
    "args": {
      "External id": 2463936,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8535
    }
  },
  {
    "ph": "s", "id": 64, "pid": 1336756, "tid": 1336756, "ts": 1514656498763.274,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498766.967, "dur": 69.353,
    "args": {
      "External id": 2463937,"Sequence number": 24740585, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8536
    }
  },
  {
    "ph": "s", "id": 63, "pid": 1336756, "tid": 1336756, "ts": 1514656498766.967,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498838.212, "dur": 3.351,
    "args": {
      "External id": 2463938,"Sequence number": 24740586, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8537
    }
  },
  {
    "ph": "s", "id": 62, "pid": 1336756, "tid": 1336756, "ts": 1514656498838.212,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498850.534, "dur": 62.852,
    "args": {
      "External id": 2463939,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498851.215, "dur": 6.190,
    "args": {
      "External id": 2463940,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8539
    }
  },
  {
    "ph": "s", "id": 61, "pid": 1336756, "tid": 1336756, "ts": 1514656498851.215,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498853.116, "dur": 3.093,
    "args": {
      "External id": 2463941,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498854.866, "dur": 1.137,
    "args": {
      "External id": 2463942,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498859.640, "dur": 53.524,
    "args": {
      "External id": 2463943,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498860.918, "dur": 5.019,
    "args": {
      "External id": 2463944,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498861.476, "dur": 4.298,
    "args": {
      "External id": 2463945,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8544
    }
  },
  {
    "ph": "s", "id": 60, "pid": 1336756, "tid": 1336756, "ts": 1514656498861.476,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498866.403, "dur": 41.201,
    "args": {
      "External id": 2463946,"Sequence number": 24740589, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8545
    }
  },
  {
    "ph": "s", "id": 59, "pid": 1336756, "tid": 1336756, "ts": 1514656498866.403,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498909.257, "dur": 3.600,
    "args": {
      "External id": 2463947,"Sequence number": 24740590, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8546
    }
  },
  {
    "ph": "s", "id": 58, "pid": 1336756, "tid": 1336756, "ts": 1514656498909.257,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498920.590, "dur": 104.408,
    "args": {
      "External id": 2463948,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498921.299, "dur": 12.642,
    "args": {
      "External id": 2463949,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8548
    }
  },
  {
    "ph": "s", "id": 57, "pid": 1336756, "tid": 1336756, "ts": 1514656498921.299,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498926.543, "dur": 6.065,
    "args": {
      "External id": 2463950,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498931.744, "dur": 0.714,
    "args": {
      "External id": 2463951,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498934.768, "dur": 89.913,
    "args": {
      "External id": 2463952,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498937.724, "dur": 5.642,
    "args": {
      "External id": 2463953,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498938.956, "dur": 4.256,
    "args": {
      "External id": 2463954,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8553
    }
  },
  {
    "ph": "s", "id": 56, "pid": 1336756, "tid": 1336756, "ts": 1514656498938.956,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498944.301, "dur": 37.619,
    "args": {
      "External id": 2463955,"Sequence number": 24740593, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8554
    }
  },
  {
    "ph": "s", "id": 55, "pid": 1336756, "tid": 1336756, "ts": 1514656498944.301,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656498983.545, "dur": 40.154,
    "args": {
      "External id": 2463956,"Sequence number": 24740594, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8555
    }
  },
  {
    "ph": "s", "id": 54, "pid": 1336756, "tid": 1336756, "ts": 1514656498983.545,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499045.330, "dur": 5.145,
    "args": {
      "External id": 2463957,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499046.565, "dur": 3.733,
    "args": {
      "External id": 2463958,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8557
    }
  },
  {
    "ph": "s", "id": 53, "pid": 1336756, "tid": 1336756, "ts": 1514656499046.565,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499057.142, "dur": 3.439,
    "args": {
      "External id": 2463959,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499057.854, "dur": 2.590,
    "args": {
      "External id": 2463960,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8559
    }
  },
  {
    "ph": "s", "id": 52, "pid": 1336756, "tid": 1336756, "ts": 1514656499057.854,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499064.324, "dur": 5.568,
    "args": {
      "External id": 2463961,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499067.229, "dur": 2.540,
    "args": {
      "External id": 2463962,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8561
    }
  },
  {
    "ph": "s", "id": 51, "pid": 1336756, "tid": 1336756, "ts": 1514656499067.229,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499101.212, "dur": 158.814,
    "args": {
      "External id": 2463963,"Sequence number": 24740598, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8562
    }
  },
  {
    "ph": "s", "id": 50, "pid": 1336756, "tid": 1336756, "ts": 1514656499101.212,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499120.365, "dur": 8.088,
    "args": {
      "External id": 2463964,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499122.575, "dur": 5.514,
    "args": {
      "External id": 2463965,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499272.471, "dur": 98.543,
    "args": {
      "External id": 2463966,"Sequence number": 24740599, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8565
    }
  },
  {
    "ph": "s", "id": 49, "pid": 1336756, "tid": 1336756, "ts": 1514656499272.471,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499285.091, "dur": 6.467,
    "args": {
      "External id": 2463967,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499286.882, "dur": 4.277,
    "args": {
      "External id": 2463968,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499398.450, "dur": 193.211,
    "args": {
      "External id": 2463969,"Sequence number": 24740600, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8568
    }
  },
  {
    "ph": "s", "id": 48, "pid": 1336756, "tid": 1336756, "ts": 1514656499398.450,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499424.625, "dur": 139.120,
    "args": {
      "External id": 2463970,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499470.425, "dur": 6.069,
    "args": {
      "External id": 2463971,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499472.359, "dur": 3.727,
    "args": {
      "External id": 2463972,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499480.661, "dur": 3.593,
    "args": {
      "External id": 2463973,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499485.556, "dur": 1.610,
    "args": {
      "External id": 2463974,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499489.812, "dur": 4.883,
    "args": {
      "External id": 2463975,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499576.057, "dur": 4.781,
    "args": {
      "External id": 2463976,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499596.763, "dur": 5.685,
    "args": {
      "External id": 2463977,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499598.131, "dur": 4.199,
    "args": {
      "External id": 2463978,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8577
    }
  },
  {
    "ph": "s", "id": 47, "pid": 1336756, "tid": 1336756, "ts": 1514656499598.131,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499613.749, "dur": 103.246,
    "args": {
      "External id": 2463979,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499614.761, "dur": 9.274,
    "args": {
      "External id": 2463980,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8579
    }
  },
  {
    "ph": "s", "id": 46, "pid": 1336756, "tid": 1336756, "ts": 1514656499614.761,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499618.989, "dur": 4.125,
    "args": {
      "External id": 2463981,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499621.357, "dur": 1.505,
    "args": {
      "External id": 2463982,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499625.540, "dur": 91.104,
    "args": {
      "External id": 2463983,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499627.522, "dur": 5.465,
    "args": {
      "External id": 2463984,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499630.033, "dur": 2.816,
    "args": {
      "External id": 2463985,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8584
    }
  },
  {
    "ph": "s", "id": 45, "pid": 1336756, "tid": 1336756, "ts": 1514656499630.033,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499634.035, "dur": 76.277,
    "args": {
      "External id": 2463986,"Sequence number": 24740604, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8585
    }
  },
  {
    "ph": "s", "id": 44, "pid": 1336756, "tid": 1336756, "ts": 1514656499634.035,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499712.202, "dur": 3.799,
    "args": {
      "External id": 2463987,"Sequence number": 24740605, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8586
    }
  },
  {
    "ph": "s", "id": 43, "pid": 1336756, "tid": 1336756, "ts": 1514656499712.202,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499749.519, "dur": 193.466,
    "args": {
      "External id": 2463988,"Sequence number": 24740606, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8587
    }
  },
  {
    "ph": "s", "id": 42, "pid": 1336756, "tid": 1336756, "ts": 1514656499749.519,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499765.668, "dur": 4.516,
    "args": {
      "External id": 2463989,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499768.478, "dur": 1.590,
    "args": {
      "External id": 2463990,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499773.723, "dur": 3.805,
    "args": {
      "External id": 2463991,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499775.261, "dur": 2.162,
    "args": {
      "External id": 2463992,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499776.076, "dur": 1.247,
    "args": {
      "External id": 2463993,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499784.365, "dur": 6.675,
    "args": {
      "External id": 2463994,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499786.290, "dur": 4.414,
    "args": {
      "External id": 2463995,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499798.372, "dur": 3.128,
    "args": {
      "External id": 2463996,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499805.153, "dur": 2.899,
    "args": {
      "External id": 2463997,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499922.373, "dur": 5.194,
    "args": {
      "External id": 2463998,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499923.500, "dur": 3.881,
    "args": {
      "External id": 2463999,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499930.427, "dur": 2.410,
    "args": {
      "External id": 2464000,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499931.729, "dur": 1.008,
    "args": {
      "External id": 2464001,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499961.167, "dur": 134.891,
    "args": {
      "External id": 2464002,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499961.905, "dur": 7.793,
    "args": {
      "External id": 2464003,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8602
    }
  },
  {
    "ph": "s", "id": 41, "pid": 1336756, "tid": 1336756, "ts": 1514656499961.905,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499965.319, "dur": 3.266,
    "args": {
      "External id": 2464004,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499967.193, "dur": 1.196,
    "args": {
      "External id": 2464005,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499970.691, "dur": 125.014,
    "args": {
      "External id": 2464006,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499972.017, "dur": 7.614,
    "args": {
      "External id": 2464007,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499974.917, "dur": 4.559,
    "args": {
      "External id": 2464008,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8607
    }
  },
  {
    "ph": "s", "id": 40, "pid": 1336756, "tid": 1336756, "ts": 1514656499974.917,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656499980.296, "dur": 107.609,
    "args": {
      "External id": 2464009,"Sequence number": 24740609, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8608
    }
  },
  {
    "ph": "s", "id": 39, "pid": 1336756, "tid": 1336756, "ts": 1514656499980.296,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500090.919, "dur": 3.960,
    "args": {
      "External id": 2464010,"Sequence number": 24740610, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8609
    }
  },
  {
    "ph": "s", "id": 38, "pid": 1336756, "tid": 1336756, "ts": 1514656500090.919,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500106.634, "dur": 75.001,
    "args": {
      "External id": 2464011,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500107.318, "dur": 11.962,
    "args": {
      "External id": 2464012,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8611
    }
  },
  {
    "ph": "s", "id": 37, "pid": 1336756, "tid": 1336756, "ts": 1514656500107.318,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500113.372, "dur": 4.341,
    "args": {
      "External id": 2464013,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500116.678, "dur": 0.873,
    "args": {
      "External id": 2464014,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500120.304, "dur": 61.065,
    "args": {
      "External id": 2464015,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500121.564, "dur": 6.500,
    "args": {
      "External id": 2464016,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500122.265, "dur": 5.574,
    "args": {
      "External id": 2464017,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8616
    }
  },
  {
    "ph": "s", "id": 36, "pid": 1336756, "tid": 1336756, "ts": 1514656500122.265,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500128.981, "dur": 46.281,
    "args": {
      "External id": 2464018,"Sequence number": 24740613, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8617
    }
  },
  {
    "ph": "s", "id": 35, "pid": 1336756, "tid": 1336756, "ts": 1514656500128.981,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500177.009, "dur": 3.995,
    "args": {
      "External id": 2464019,"Sequence number": 24740614, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8618
    }
  },
  {
    "ph": "s", "id": 34, "pid": 1336756, "tid": 1336756, "ts": 1514656500177.009,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500203.409, "dur": 137.886,
    "args": {
      "External id": 2464020,"Sequence number": 24740615, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8619
    }
  },
  {
    "ph": "s", "id": 33, "pid": 1336756, "tid": 1336756, "ts": 1514656500203.409,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500239.558, "dur": 4.235,
    "args": {
      "External id": 2464021,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500272.389, "dur": 56.166,
    "args": {
      "External id": 2464022,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500273.190, "dur": 4.637,
    "args": {
      "External id": 2464023,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500274.253, "dur": 2.553,
    "args": {
      "External id": 2464024,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500275.707, "dur": 0.873,
    "args": {
      "External id": 2464025,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500278.530, "dur": 49.740,
    "args": {
      "External id": 2464026,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500279.846, "dur": 3.813,
    "args": {
      "External id": 2464027,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500281.959, "dur": 1.595,
    "args": {
      "External id": 2464028,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500284.316, "dur": 40.288,
    "args": {
      "External id": 2464029,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500326.581, "dur": 1.046,
    "args": {
      "External id": 2464030,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500349.488, "dur": 22.611,
    "args": {
      "External id": 2464031,"Sequence number": 24740616, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8630
    }
  },
  {
    "ph": "s", "id": 32, "pid": 1336756, "tid": 1336756, "ts": 1514656500349.488,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500390.944, "dur": 41.632,
    "args": {
      "External id": 2464032,"Sequence number": 24740617, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "-2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8631
    }
  },
  {
    "ph": "s", "id": 31, "pid": 1336756, "tid": 1336756, "ts": 1514656500390.944,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500399.284, "dur": 29.128,
    "args": {
      "External id": 2464033,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500429.915, "dur": 1.322,
    "args": {
      "External id": 2464034,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 8192], []], "Ev Idx": 8633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500471.815, "dur": 38.716,
    "args": {
      "External id": 2464035,"Record function id": 0, "Ev Idx": 8634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 2/0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500512.125, "dur": 203.349,
    "args": {
      "External id": 2464036,"Record function id": 0, "Ev Idx": 8635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500565.149, "dur": 142.460,
    "args": {
      "External id": 2464037,"Sequence number": 24740618, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1], [33554432, 8192, 2048, 1]], "Input Dims": [[2048], [16, 4096, 4, 2048]], "Ev Idx": 8636
    }
  },
  {
    "ph": "s", "id": 30, "pid": 1336756, "tid": 1336756, "ts": 1514656500565.149,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500632.249, "dur": 37.382,
    "args": {
      "External id": 2464038,"kernel_hash": "cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/d2/cd26ogptnjelaibpl2zumx5nxdttspqepvjkh7lpmsxbzrrexbxm.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [262144, 2048], [2048], [262144], [], [], [], [], [], [], [], [], []], "Ev Idx": 8637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500808.606, "dur": 42.042,
    "args": {
      "External id": 2464039,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500811.652, "dur": 5.391,
    "args": {
      "External id": 2464040,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500820.306, "dur": 30.024,
    "args": {
      "External id": 2464041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500823.861, "dur": 26.047,
    "args": {
      "External id": 2464042,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500855.083, "dur": 20.706,
    "args": {
      "External id": 2464043,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500856.281, "dur": 2.914,
    "args": {
      "External id": 2464044,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500859.765, "dur": 15.721,
    "args": {
      "External id": 2464045,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500862.337, "dur": 12.696,
    "args": {
      "External id": 2464046,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500878.707, "dur": 15.000,
    "args": {
      "External id": 2464047,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500879.294, "dur": 2.478,
    "args": {
      "External id": 2464048,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500882.388, "dur": 11.062,
    "args": {
      "External id": 2464049,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500883.234, "dur": 9.809,
    "args": {
      "External id": 2464050,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500904.498, "dur": 0.615,
    "args": {
      "External id": 2464051,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 8650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500913.479, "dur": 10.958,
    "args": {
      "External id": 2464052,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "5", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500921.093, "dur": 1.789,
    "args": {
      "External id": 2464053,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500930.588, "dur": 6.346,
    "args": {
      "External id": 2464054,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500933.999, "dur": 1.074,
    "args": {
      "External id": 2464055,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500938.081, "dur": 3.810,
    "args": {
      "External id": 2464056,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500939.993, "dur": 0.920,
    "args": {
      "External id": 2464057,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500943.279, "dur": 3.174,
    "args": {
      "External id": 2464058,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500945.331, "dur": 0.615,
    "args": {
      "External id": 2464059,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500949.685, "dur": 2.975,
    "args": {
      "External id": 2464060,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500951.241, "dur": 0.809,
    "args": {
      "External id": 2464061,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500953.917, "dur": 3.147,
    "args": {
      "External id": 2464062,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500955.333, "dur": 0.938,
    "args": {
      "External id": 2464063,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500958.141, "dur": 2.714,
    "args": {
      "External id": 2464064,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4096, 4], [], [], [], []], "Ev Idx": 8663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500959.760, "dur": 0.563,
    "args": {
      "External id": 2464065,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500964.242, "dur": 4.374,
    "args": {
      "External id": 2464066,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4096, 4], [], []], "Ev Idx": 8665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500967.326, "dur": 0.497,
    "args": {
      "External id": 2464067,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500972.877, "dur": 3.014,
    "args": {
      "External id": 2464068,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 8667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500974.611, "dur": 0.653,
    "args": {
      "External id": 2464069,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656500979.730, "dur": 47.531,
    "args": {
      "External id": 2464070,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 8669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501024.410, "dur": 0.948,
    "args": {
      "External id": 2464071,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501029.296, "dur": 3.339,
    "args": {
      "External id": 2464072,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 8671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501031.193, "dur": 0.806,
    "args": {
      "External id": 2464073,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 8672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501036.944, "dur": 6.248,
    "args": {
      "External id": 2464074,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8673
    }
  },
  {
    "ph": "s", "id": 29, "pid": 1336756, "tid": 1336756, "ts": 1514656501036.944,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501040.502, "dur": 0.873,
    "args": {
      "External id": 2464075,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501046.156, "dur": 4.812,
    "args": {
      "External id": 2464076,"Sequence number": 24740620, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8675
    }
  },
  {
    "ph": "s", "id": 28, "pid": 1336756, "tid": 1336756, "ts": 1514656501046.156,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501049.508, "dur": 0.736,
    "args": {
      "External id": 2464077,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501052.222, "dur": 5.985,
    "args": {
      "External id": 2464078,"Sequence number": 24740621, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 8677
    }
  },
  {
    "ph": "s", "id": 27, "pid": 1336756, "tid": 1336756, "ts": 1514656501052.222,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501056.483, "dur": 0.897,
    "args": {
      "External id": 2464079,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501059.375, "dur": 4.386,
    "args": {
      "External id": 2464080,"Sequence number": 24740622, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 8679
    }
  },
  {
    "ph": "s", "id": 26, "pid": 1336756, "tid": 1336756, "ts": 1514656501059.375,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501062.107, "dur": 0.939,
    "args": {
      "External id": 2464081,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 8680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501068.666, "dur": 45.698,
    "args": {
      "External id": 2464082,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501072.815, "dur": 41.194,
    "args": {
      "External id": 2464083,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501075.904, "dur": 8.082,
    "args": {
      "External id": 2464084,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 8683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501077.888, "dur": 5.453,
    "args": {
      "External id": 2464085,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501085.433, "dur": 28.093,
    "args": {
      "External id": 2464086,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 8685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501139.401, "dur": 4.593,
    "args": {
      "External id": 2464087,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8686
    }
  },
  {
    "ph": "s", "id": 25, "pid": 1336756, "tid": 1336756, "ts": 1514656501139.401,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501146.109, "dur": 1.437,
    "args": {
      "External id": 2464088,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501174.762, "dur": 71990.338,
    "args": {
      "External id": 2464089,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 8688
    }
  },
  {
    "ph": "s", "id": 24, "pid": 1336756, "tid": 1336756, "ts": 1514656501174.762,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501189.239, "dur": 26.164,
    "args": {
      "External id": 2464090,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501189.738, "dur": 25.405,
    "args": {
      "External id": 2464091,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501190.968, "dur": 5.323,
    "args": {
      "External id": 2464092,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501192.471, "dur": 3.394,
    "args": {
      "External id": 2464093,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501196.927, "dur": 17.840,
    "args": {
      "External id": 2464094,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 8693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501233.165, "dur": 25.484,
    "args": {
      "External id": 2464095,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501234.581, "dur": 6.221,
    "args": {
      "External id": 2464096,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501236.375, "dur": 4.074,
    "args": {
      "External id": 2464097,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501241.964, "dur": 16.472,
    "args": {
      "External id": 2464098,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 8697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501242.992, "dur": 15.083,
    "args": {
      "External id": 2464099,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501262.243, "dur": 20.932,
    "args": {
      "External id": 2464100,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501263.215, "dur": 5.872,
    "args": {
      "External id": 2464101,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501266.128, "dur": 2.718,
    "args": {
      "External id": 2464102,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501269.625, "dur": 13.320,
    "args": {
      "External id": 2464103,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501270.007, "dur": 12.533,
    "args": {
      "External id": 2464104,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 8703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501287.509, "dur": 25.781,
    "args": {
      "External id": 2464105,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501289.018, "dur": 3.020,
    "args": {
      "External id": 2464106,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501300.546, "dur": 12.460,
    "args": {
      "External id": 2464107,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 8706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501300.952, "dur": 11.485,
    "args": {
      "External id": 2464108,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501322.771, "dur": 34.044,
    "args": {
      "External id": 2464109,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501361.871, "dur": 60.830,
    "args": {
      "External id": 2464110,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501365.943, "dur": 56.295,
    "args": {
      "External id": 2464111,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501373.043, "dur": 1.224,
    "args": {
      "External id": 2464112,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 8711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501376.500, "dur": 25.744,
    "args": {
      "External id": 2464113,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501378.452, "dur": 23.568,
    "args": {
      "External id": 2464114,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 8713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501380.873, "dur": 4.838,
    "args": {
      "External id": 2464115,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501386.876, "dur": 14.628,
    "args": {
      "External id": 2464116,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 8715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501428.061, "dur": 65934.481,
    "args": {
      "External id": 2464117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1336756,
    "ts": 1514656501430.764, "dur": 65930.503,
    "args": {
      "External id": 2464118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567376.847, "dur": 9.106,
    "args": {
      "External id": 2464119,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567382.633, "dur": 1.269,
    "args": {
      "External id": 2464120,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567391.373, "dur": 114.568,
    "args": {
      "External id": 2464121,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567395.202, "dur": 6.302,
    "args": {
      "External id": 2464122,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567397.499, "dur": 2.780,
    "args": {
      "External id": 2464123,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567399.400, "dur": 0.606,
    "args": {
      "External id": 2464124,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567403.241, "dur": 101.669,
    "args": {
      "External id": 2464125,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567405.089, "dur": 99.071,
    "args": {
      "External id": 2464126,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567509.532, "dur": 4.540,
    "args": {
      "External id": 2464127,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567511.450, "dur": 1.006,
    "args": {
      "External id": 2464128,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567525.392, "dur": 16.938,
    "args": {
      "External id": 2464129,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567556.933, "dur": 8.564,
    "args": {
      "External id": 2464130,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567560.110, "dur": 5.025,
    "args": {
      "External id": 2464131,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567723.948, "dur": 286.386,
    "args": {
      "External id": 2464132,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567729.080, "dur": 2.834,
    "args": {
      "External id": 2464133,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567734.888, "dur": 245.894,
    "args": {
      "External id": 2464134,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567740.284, "dur": 0.837,
    "args": {
      "External id": 2464135,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567745.300, "dur": 29.792,
    "args": {
      "External id": 2464136,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567777.768, "dur": 6.617,
    "args": {
      "External id": 2464137,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567782.684, "dur": 1.180,
    "args": {
      "External id": 2464138,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567786.954, "dur": 27.985,
    "args": {
      "External id": 2464139,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567788.023, "dur": 1.584,
    "args": {
      "External id": 2464140,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567792.138, "dur": 22.513,
    "args": {
      "External id": 2464141,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567797.411, "dur": 3.717,
    "args": {
      "External id": 2464142,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567817.942, "dur": 26.281,
    "args": {
      "External id": 2464143,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567847.559, "dur": 23.154,
    "args": {
      "External id": 2464144,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567877.684, "dur": 17.922,
    "args": {
      "External id": 2464145,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567898.713, "dur": 19.746,
    "args": {
      "External id": 2464146,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567921.172, "dur": 22.962,
    "args": {
      "External id": 2464147,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567923.900, "dur": 1.661,
    "args": {
      "External id": 2464148,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567927.538, "dur": 0.742,
    "args": {
      "External id": 2464149,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567947.426, "dur": 16.181,
    "args": {
      "External id": 2464150,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656567968.965, "dur": 10.527,
    "args": {
      "External id": 2464151,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568020.047, "dur": 2.773,
    "args": {
      "External id": 2464152,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568030.922, "dur": 4.084,
    "args": {
      "External id": 2464153,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568033.240, "dur": 0.809,
    "args": {
      "External id": 2464154,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568127.968, "dur": 74.246,
    "args": {
      "External id": 2464155,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568207.696, "dur": 4.586,
    "args": {
      "External id": 2464156,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568209.937, "dur": 0.974,
    "args": {
      "External id": 2464157,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568214.397, "dur": 35.147,
    "args": {
      "External id": 2464158,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568257.312, "dur": 6.445,
    "args": {
      "External id": 2464159,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568259.666, "dur": 3.448,
    "args": {
      "External id": 2464160,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568261.741, "dur": 1.113,
    "args": {
      "External id": 2464161,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568268.544, "dur": 49.129,
    "args": {
      "External id": 2464162,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568269.965, "dur": 47.112,
    "args": {
      "External id": 2464163,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568323.252, "dur": 18.137,
    "args": {
      "External id": 2464164,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568347.458, "dur": 5.926,
    "args": {
      "External id": 2464165,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568351.564, "dur": 0.972,
    "args": {
      "External id": 2464166,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568357.569, "dur": 53.019,
    "args": {
      "External id": 2464167,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568358.496, "dur": 4.091,
    "args": {
      "External id": 2464168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568359.258, "dur": 2.747,
    "args": {
      "External id": 2464169,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568360.244, "dur": 1.600,
    "args": {
      "External id": 2464170,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568363.242, "dur": 46.819,
    "args": {
      "External id": 2464171,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568365.460, "dur": 44.077,
    "args": {
      "External id": 2464172,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568414.508, "dur": 4.365,
    "args": {
      "External id": 2464173,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568416.481, "dur": 0.779,
    "args": {
      "External id": 2464174,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568424.853, "dur": 1.771,
    "args": {
      "External id": 2464175,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568434.631, "dur": 9.735,
    "args": {
      "External id": 2464176,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568438.286, "dur": 5.738,
    "args": {
      "External id": 2464177,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568551.821, "dur": 186.696,
    "args": {
      "External id": 2464178,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568554.362, "dur": 3.084,
    "args": {
      "External id": 2464179,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568559.321, "dur": 178.772,
    "args": {
      "External id": 2464180,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568561.614, "dur": 0.300,
    "args": {
      "External id": 2464181,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568563.509, "dur": 25.101,
    "args": {
      "External id": 2464182,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568590.448, "dur": 3.251,
    "args": {
      "External id": 2464183,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568592.460, "dur": 0.940,
    "args": {
      "External id": 2464184,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568594.832, "dur": 26.975,
    "args": {
      "External id": 2464185,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568598.337, "dur": 2.060,
    "args": {
      "External id": 2464186,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568601.623, "dur": 19.895,
    "args": {
      "External id": 2464187,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568606.809, "dur": 2.941,
    "args": {
      "External id": 2464188,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568623.404, "dur": 21.706,
    "args": {
      "External id": 2464189,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568647.005, "dur": 13.529,
    "args": {
      "External id": 2464190,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568663.246, "dur": 13.195,
    "args": {
      "External id": 2464191,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568678.200, "dur": 12.061,
    "args": {
      "External id": 2464192,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568691.752, "dur": 20.358,
    "args": {
      "External id": 2464193,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568693.750, "dur": 1.240,
    "args": {
      "External id": 2464194,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568698.127, "dur": 0.747,
    "args": {
      "External id": 2464195,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568713.660, "dur": 11.381,
    "args": {
      "External id": 2464196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568726.231, "dur": 10.827,
    "args": {
      "External id": 2464197,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568745.916, "dur": 2.067,
    "args": {
      "External id": 2464198,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568756.651, "dur": 3.919,
    "args": {
      "External id": 2464199,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568759.103, "dur": 0.672,
    "args": {
      "External id": 2464200,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568829.166, "dur": 55.680,
    "args": {
      "External id": 2464201,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568889.560, "dur": 7.464,
    "args": {
      "External id": 2464202,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568894.464, "dur": 1.320,
    "args": {
      "External id": 2464203,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568898.569, "dur": 24.823,
    "args": {
      "External id": 2464204,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568927.751, "dur": 5.075,
    "args": {
      "External id": 2464205,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568929.282, "dur": 2.927,
    "args": {
      "External id": 2464206,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568931.089, "dur": 0.911,
    "args": {
      "External id": 2464207,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568935.622, "dur": 43.083,
    "args": {
      "External id": 2464208,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656568938.272, "dur": 39.872,
    "args": {
      "External id": 2464209,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569017.258, "dur": 17.898,
    "args": {
      "External id": 2464210,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569042.475, "dur": 4.472,
    "args": {
      "External id": 2464211,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569044.944, "dur": 1.006,
    "args": {
      "External id": 2464212,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569051.145, "dur": 59.702,
    "args": {
      "External id": 2464213,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569052.252, "dur": 7.370,
    "args": {
      "External id": 2464214,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569053.470, "dur": 5.479,
    "args": {
      "External id": 2464215,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569056.251, "dur": 2.491,
    "args": {
      "External id": 2464216,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569060.508, "dur": 49.782,
    "args": {
      "External id": 2464217,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569061.569, "dur": 48.102,
    "args": {
      "External id": 2464218,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569114.960, "dur": 3.775,
    "args": {
      "External id": 2464219,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569116.731, "dur": 0.905,
    "args": {
      "External id": 2464220,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569125.082, "dur": 1.535,
    "args": {
      "External id": 2464221,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569134.152, "dur": 8.028,
    "args": {
      "External id": 2464222,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569138.364, "dur": 3.581,
    "args": {
      "External id": 2464223,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569230.384, "dur": 314.482,
    "args": {
      "External id": 2464224,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569232.528, "dur": 2.200,
    "args": {
      "External id": 2464225,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569236.608, "dur": 307.740,
    "args": {
      "External id": 2464226,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569238.351, "dur": 0.283,
    "args": {
      "External id": 2464227,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569241.814, "dur": 19.512,
    "args": {
      "External id": 2464228,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569262.986, "dur": 3.783,
    "args": {
      "External id": 2464229,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569265.447, "dur": 1.100,
    "args": {
      "External id": 2464230,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569267.792, "dur": 24.994,
    "args": {
      "External id": 2464231,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569270.670, "dur": 3.321,
    "args": {
      "External id": 2464232,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569275.580, "dur": 16.882,
    "args": {
      "External id": 2464233,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569278.623, "dur": 2.802,
    "args": {
      "External id": 2464234,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569294.407, "dur": 20.821,
    "args": {
      "External id": 2464235,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569316.894, "dur": 19.022,
    "args": {
      "External id": 2464236,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569338.801, "dur": 41.494,
    "args": {
      "External id": 2464237,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569381.882, "dur": 36.257,
    "args": {
      "External id": 2464238,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569419.692, "dur": 44.715,
    "args": {
      "External id": 2464239,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569421.813, "dur": 1.540,
    "args": {
      "External id": 2464240,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569426.526, "dur": 1.040,
    "args": {
      "External id": 2464241,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569466.002, "dur": 40.616,
    "args": {
      "External id": 2464242,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569507.784, "dur": 34.921,
    "args": {
      "External id": 2464243,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569553.270, "dur": 2.222,
    "args": {
      "External id": 2464244,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569564.810, "dur": 3.694,
    "args": {
      "External id": 2464245,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569567.142, "dur": 0.581,
    "args": {
      "External id": 2464246,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569639.712, "dur": 54.225,
    "args": {
      "External id": 2464247,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569700.485, "dur": 4.639,
    "args": {
      "External id": 2464248,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569703.122, "dur": 0.945,
    "args": {
      "External id": 2464249,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569706.724, "dur": 24.879,
    "args": {
      "External id": 2464250,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569736.404, "dur": 6.259,
    "args": {
      "External id": 2464251,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569737.606, "dur": 4.483,
    "args": {
      "External id": 2464252,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569739.163, "dur": 2.693,
    "args": {
      "External id": 2464253,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569747.005, "dur": 41.365,
    "args": {
      "External id": 2464254,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569748.321, "dur": 39.439,
    "args": {
      "External id": 2464255,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569791.984, "dur": 14.242,
    "args": {
      "External id": 2464256,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569846.219, "dur": 3.961,
    "args": {
      "External id": 2464257,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569848.764, "dur": 0.630,
    "args": {
      "External id": 2464258,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569854.091, "dur": 51.571,
    "args": {
      "External id": 2464259,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569854.842, "dur": 5.169,
    "args": {
      "External id": 2464260,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569857.523, "dur": 1.874,
    "args": {
      "External id": 2464261,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569858.474, "dur": 0.736,
    "args": {
      "External id": 2464262,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569860.572, "dur": 44.714,
    "args": {
      "External id": 2464263,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569861.217, "dur": 43.406,
    "args": {
      "External id": 2464264,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569909.419, "dur": 3.776,
    "args": {
      "External id": 2464265,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569911.453, "dur": 0.689,
    "args": {
      "External id": 2464266,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569920.548, "dur": 1.499,
    "args": {
      "External id": 2464267,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569929.566, "dur": 7.021,
    "args": {
      "External id": 2464268,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656569931.106, "dur": 5.195,
    "args": {
      "External id": 2464269,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570068.023, "dur": 183.492,
    "args": {
      "External id": 2464270,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570070.458, "dur": 3.285,
    "args": {
      "External id": 2464271,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570075.791, "dur": 175.308,
    "args": {
      "External id": 2464272,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570079.040, "dur": 0.460,
    "args": {
      "External id": 2464273,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570080.646, "dur": 25.807,
    "args": {
      "External id": 2464274,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570108.147, "dur": 2.944,
    "args": {
      "External id": 2464275,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570109.945, "dur": 0.845,
    "args": {
      "External id": 2464276,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570112.009, "dur": 23.286,
    "args": {
      "External id": 2464277,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570113.407, "dur": 1.491,
    "args": {
      "External id": 2464278,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570116.012, "dur": 19.003,
    "args": {
      "External id": 2464279,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570121.044, "dur": 2.531,
    "args": {
      "External id": 2464280,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570136.768, "dur": 20.818,
    "args": {
      "External id": 2464281,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570159.075, "dur": 13.740,
    "args": {
      "External id": 2464282,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570177.415, "dur": 13.649,
    "args": {
      "External id": 2464283,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570192.865, "dur": 12.380,
    "args": {
      "External id": 2464284,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570206.854, "dur": 17.670,
    "args": {
      "External id": 2464285,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570208.370, "dur": 1.217,
    "args": {
      "External id": 2464286,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570211.597, "dur": 0.857,
    "args": {
      "External id": 2464287,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570225.768, "dur": 11.845,
    "args": {
      "External id": 2464288,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570238.808, "dur": 11.193,
    "args": {
      "External id": 2464289,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570260.388, "dur": 2.076,
    "args": {
      "External id": 2464290,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570272.290, "dur": 3.722,
    "args": {
      "External id": 2464291,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570274.613, "dur": 0.690,
    "args": {
      "External id": 2464292,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570342.914, "dur": 54.117,
    "args": {
      "External id": 2464293,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570401.927, "dur": 4.685,
    "args": {
      "External id": 2464294,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570404.470, "dur": 0.931,
    "args": {
      "External id": 2464295,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570408.183, "dur": 25.191,
    "args": {
      "External id": 2464296,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570438.094, "dur": 6.579,
    "args": {
      "External id": 2464297,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570441.253, "dur": 2.735,
    "args": {
      "External id": 2464298,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570443.000, "dur": 0.804,
    "args": {
      "External id": 2464299,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570447.631, "dur": 40.087,
    "args": {
      "External id": 2464300,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570448.700, "dur": 38.358,
    "args": {
      "External id": 2464301,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570491.435, "dur": 13.525,
    "args": {
      "External id": 2464302,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570510.711, "dur": 5.856,
    "args": {
      "External id": 2464303,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570514.948, "dur": 0.841,
    "args": {
      "External id": 2464304,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570520.499, "dur": 70.549,
    "args": {
      "External id": 2464305,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570521.194, "dur": 5.070,
    "args": {
      "External id": 2464306,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570522.139, "dur": 3.563,
    "args": {
      "External id": 2464307,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570523.179, "dur": 2.319,
    "args": {
      "External id": 2464308,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570543.079, "dur": 47.155,
    "args": {
      "External id": 2464309,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570544.233, "dur": 45.263,
    "args": {
      "External id": 2464310,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570596.261, "dur": 4.058,
    "args": {
      "External id": 2464311,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570598.184, "dur": 0.993,
    "args": {
      "External id": 2464312,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570608.006, "dur": 1.711,
    "args": {
      "External id": 2464313,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570617.670, "dur": 5.291,
    "args": {
      "External id": 2464314,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570619.090, "dur": 3.554,
    "args": {
      "External id": 2464315,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570705.541, "dur": 173.742,
    "args": {
      "External id": 2464316,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570707.722, "dur": 2.095,
    "args": {
      "External id": 2464317,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570713.217, "dur": 165.674,
    "args": {
      "External id": 2464318,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570714.462, "dur": 0.434,
    "args": {
      "External id": 2464319,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570718.261, "dur": 20.753,
    "args": {
      "External id": 2464320,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570740.733, "dur": 3.340,
    "args": {
      "External id": 2464321,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570742.885, "dur": 0.943,
    "args": {
      "External id": 2464322,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570744.857, "dur": 23.523,
    "args": {
      "External id": 2464323,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570746.145, "dur": 3.146,
    "args": {
      "External id": 2464324,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570750.633, "dur": 17.222,
    "args": {
      "External id": 2464325,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570753.463, "dur": 2.466,
    "args": {
      "External id": 2464326,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570771.336, "dur": 19.008,
    "args": {
      "External id": 2464327,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570791.864, "dur": 12.673,
    "args": {
      "External id": 2464328,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570807.344, "dur": 12.446,
    "args": {
      "External id": 2464329,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570821.414, "dur": 11.462,
    "args": {
      "External id": 2464330,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570834.294, "dur": 19.121,
    "args": {
      "External id": 2464331,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570836.301, "dur": 1.604,
    "args": {
      "External id": 2464332,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570840.145, "dur": 0.762,
    "args": {
      "External id": 2464333,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570854.566, "dur": 10.994,
    "args": {
      "External id": 2464334,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570868.257, "dur": 9.724,
    "args": {
      "External id": 2464335,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570885.098, "dur": 1.583,
    "args": {
      "External id": 2464336,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570894.604, "dur": 3.271,
    "args": {
      "External id": 2464337,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570896.469, "dur": 0.700,
    "args": {
      "External id": 2464338,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656570960.082, "dur": 85.613,
    "args": {
      "External id": 2464339,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571052.630, "dur": 5.405,
    "args": {
      "External id": 2464340,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571055.609, "dur": 1.026,
    "args": {
      "External id": 2464341,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571059.790, "dur": 23.509,
    "args": {
      "External id": 2464342,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571090.643, "dur": 6.615,
    "args": {
      "External id": 2464343,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571092.177, "dur": 4.461,
    "args": {
      "External id": 2464344,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571093.985, "dur": 2.403,
    "args": {
      "External id": 2464345,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571100.369, "dur": 43.751,
    "args": {
      "External id": 2464346,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571101.309, "dur": 42.147,
    "args": {
      "External id": 2464347,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571148.335, "dur": 13.789,
    "args": {
      "External id": 2464348,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571168.233, "dur": 5.397,
    "args": {
      "External id": 2464349,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571172.190, "dur": 0.585,
    "args": {
      "External id": 2464350,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571177.242, "dur": 47.944,
    "args": {
      "External id": 2464351,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571178.122, "dur": 3.811,
    "args": {
      "External id": 2464352,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571178.817, "dur": 2.632,
    "args": {
      "External id": 2464353,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571180.291, "dur": 0.984,
    "args": {
      "External id": 2464354,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571182.538, "dur": 42.245,
    "args": {
      "External id": 2464355,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571185.044, "dur": 39.237,
    "args": {
      "External id": 2464356,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571228.949, "dur": 3.667,
    "args": {
      "External id": 2464357,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571230.799, "dur": 0.830,
    "args": {
      "External id": 2464358,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571238.020, "dur": 1.642,
    "args": {
      "External id": 2464359,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571246.744, "dur": 8.124,
    "args": {
      "External id": 2464360,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571250.685, "dur": 3.938,
    "args": {
      "External id": 2464361,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571343.552, "dur": 205.033,
    "args": {
      "External id": 2464362,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571345.631, "dur": 6.236,
    "args": {
      "External id": 2464363,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571355.443, "dur": 192.564,
    "args": {
      "External id": 2464364,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571356.550, "dur": 0.257,
    "args": {
      "External id": 2464365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571357.838, "dur": 19.482,
    "args": {
      "External id": 2464366,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571378.882, "dur": 10.578,
    "args": {
      "External id": 2464367,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571386.838, "dur": 2.344,
    "args": {
      "External id": 2464368,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571390.435, "dur": 22.846,
    "args": {
      "External id": 2464369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571393.868, "dur": 1.725,
    "args": {
      "External id": 2464370,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571396.775, "dur": 16.261,
    "args": {
      "External id": 2464371,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571399.165, "dur": 2.919,
    "args": {
      "External id": 2464372,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571414.656, "dur": 19.330,
    "args": {
      "External id": 2464373,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571435.465, "dur": 12.247,
    "args": {
      "External id": 2464374,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571450.387, "dur": 12.679,
    "args": {
      "External id": 2464375,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571464.718, "dur": 12.178,
    "args": {
      "External id": 2464376,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571480.304, "dur": 22.045,
    "args": {
      "External id": 2464377,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571481.996, "dur": 1.506,
    "args": {
      "External id": 2464378,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571487.188, "dur": 1.902,
    "args": {
      "External id": 2464379,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571503.765, "dur": 11.192,
    "args": {
      "External id": 2464380,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571515.799, "dur": 30.392,
    "args": {
      "External id": 2464381,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571557.223, "dur": 1.982,
    "args": {
      "External id": 2464382,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571568.596, "dur": 3.587,
    "args": {
      "External id": 2464383,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571570.827, "dur": 0.656,
    "args": {
      "External id": 2464384,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571637.734, "dur": 52.620,
    "args": {
      "External id": 2464385,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571696.839, "dur": 4.220,
    "args": {
      "External id": 2464386,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571699.127, "dur": 1.043,
    "args": {
      "External id": 2464387,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571702.547, "dur": 24.842,
    "args": {
      "External id": 2464388,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571731.115, "dur": 5.358,
    "args": {
      "External id": 2464389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571732.460, "dur": 3.452,
    "args": {
      "External id": 2464390,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571734.391, "dur": 1.347,
    "args": {
      "External id": 2464391,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571740.400, "dur": 39.465,
    "args": {
      "External id": 2464392,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571741.505, "dur": 37.898,
    "args": {
      "External id": 2464393,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571783.451, "dur": 14.011,
    "args": {
      "External id": 2464394,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571802.495, "dur": 4.080,
    "args": {
      "External id": 2464395,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571804.298, "dur": 1.490,
    "args": {
      "External id": 2464396,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571810.136, "dur": 48.772,
    "args": {
      "External id": 2464397,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571810.949, "dur": 5.058,
    "args": {
      "External id": 2464398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571813.420, "dur": 2.090,
    "args": {
      "External id": 2464399,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571814.630, "dur": 0.764,
    "args": {
      "External id": 2464400,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571816.652, "dur": 41.825,
    "args": {
      "External id": 2464401,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571817.531, "dur": 40.327,
    "args": {
      "External id": 2464402,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571862.544, "dur": 3.615,
    "args": {
      "External id": 2464403,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571864.355, "dur": 0.763,
    "args": {
      "External id": 2464404,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571870.515, "dur": 1.256,
    "args": {
      "External id": 2464405,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571880.075, "dur": 5.933,
    "args": {
      "External id": 2464406,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571881.669, "dur": 4.092,
    "args": {
      "External id": 2464407,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571960.610, "dur": 212.363,
    "args": {
      "External id": 2464408,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571964.471, "dur": 2.051,
    "args": {
      "External id": 2464409,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571967.843, "dur": 204.610,
    "args": {
      "External id": 2464410,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571969.494, "dur": 0.480,
    "args": {
      "External id": 2464411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656571970.966, "dur": 55.497,
    "args": {
      "External id": 2464412,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572031.006, "dur": 4.752,
    "args": {
      "External id": 2464413,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572034.564, "dur": 0.900,
    "args": {
      "External id": 2464414,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572036.631, "dur": 22.711,
    "args": {
      "External id": 2464415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572038.301, "dur": 1.864,
    "args": {
      "External id": 2464416,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572041.250, "dur": 17.851,
    "args": {
      "External id": 2464417,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572044.185, "dur": 2.419,
    "args": {
      "External id": 2464418,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572060.505, "dur": 19.326,
    "args": {
      "External id": 2464419,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572081.191, "dur": 13.383,
    "args": {
      "External id": 2464420,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572097.318, "dur": 12.440,
    "args": {
      "External id": 2464421,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572110.992, "dur": 11.766,
    "args": {
      "External id": 2464422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572125.837, "dur": 20.626,
    "args": {
      "External id": 2464423,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572129.797, "dur": 1.312,
    "args": {
      "External id": 2464424,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572133.049, "dur": 1.124,
    "args": {
      "External id": 2464425,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572147.733, "dur": 10.951,
    "args": {
      "External id": 2464426,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572159.656, "dur": 11.755,
    "args": {
      "External id": 2464427,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572180.222, "dur": 2.221,
    "args": {
      "External id": 2464428,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572190.941, "dur": 3.592,
    "args": {
      "External id": 2464429,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572193.160, "dur": 0.546,
    "args": {
      "External id": 2464430,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572259.555, "dur": 51.569,
    "args": {
      "External id": 2464431,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572315.804, "dur": 4.763,
    "args": {
      "External id": 2464432,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572318.433, "dur": 1.217,
    "args": {
      "External id": 2464433,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572321.763, "dur": 22.452,
    "args": {
      "External id": 2464434,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572348.545, "dur": 6.528,
    "args": {
      "External id": 2464435,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572349.912, "dur": 4.549,
    "args": {
      "External id": 2464436,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572353.021, "dur": 1.229,
    "args": {
      "External id": 2464437,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572357.312, "dur": 38.136,
    "args": {
      "External id": 2464438,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572358.378, "dur": 36.574,
    "args": {
      "External id": 2464439,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572398.779, "dur": 12.404,
    "args": {
      "External id": 2464440,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572416.254, "dur": 3.904,
    "args": {
      "External id": 2464441,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572418.269, "dur": 1.195,
    "args": {
      "External id": 2464442,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572423.566, "dur": 48.161,
    "args": {
      "External id": 2464443,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572426.239, "dur": 3.378,
    "args": {
      "External id": 2464444,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572427.014, "dur": 2.081,
    "args": {
      "External id": 2464445,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572428.306, "dur": 0.679,
    "args": {
      "External id": 2464446,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572430.237, "dur": 41.116,
    "args": {
      "External id": 2464447,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572430.937, "dur": 39.934,
    "args": {
      "External id": 2464448,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572475.310, "dur": 3.384,
    "args": {
      "External id": 2464449,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572477.038, "dur": 0.725,
    "args": {
      "External id": 2464450,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572485.188, "dur": 1.379,
    "args": {
      "External id": 2464451,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572493.105, "dur": 5.237,
    "args": {
      "External id": 2464452,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572494.718, "dur": 3.369,
    "args": {
      "External id": 2464453,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572595.103, "dur": 172.634,
    "args": {
      "External id": 2464454,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572598.686, "dur": 2.884,
    "args": {
      "External id": 2464455,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572604.613, "dur": 162.554,
    "args": {
      "External id": 2464456,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572606.195, "dur": 0.268,
    "args": {
      "External id": 2464457,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572607.829, "dur": 20.427,
    "args": {
      "External id": 2464458,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572629.918, "dur": 4.869,
    "args": {
      "External id": 2464459,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572633.482, "dur": 1.058,
    "args": {
      "External id": 2464460,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572635.564, "dur": 19.668,
    "args": {
      "External id": 2464461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572636.735, "dur": 1.499,
    "args": {
      "External id": 2464462,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572639.478, "dur": 15.490,
    "args": {
      "External id": 2464463,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572642.061, "dur": 2.252,
    "args": {
      "External id": 2464464,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572658.761, "dur": 20.458,
    "args": {
      "External id": 2464465,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572680.685, "dur": 11.954,
    "args": {
      "External id": 2464466,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572695.538, "dur": 12.776,
    "args": {
      "External id": 2464467,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572709.714, "dur": 11.355,
    "args": {
      "External id": 2464468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572722.568, "dur": 17.731,
    "args": {
      "External id": 2464469,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572724.480, "dur": 1.026,
    "args": {
      "External id": 2464470,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572727.305, "dur": 0.900,
    "args": {
      "External id": 2464471,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572741.711, "dur": 11.099,
    "args": {
      "External id": 2464472,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572755.683, "dur": 10.482,
    "args": {
      "External id": 2464473,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572773.998, "dur": 1.939,
    "args": {
      "External id": 2464474,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572784.077, "dur": 3.475,
    "args": {
      "External id": 2464475,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572786.205, "dur": 0.603,
    "args": {
      "External id": 2464476,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572848.471, "dur": 53.327,
    "args": {
      "External id": 2464477,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572906.934, "dur": 4.510,
    "args": {
      "External id": 2464478,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572909.505, "dur": 1.055,
    "args": {
      "External id": 2464479,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572912.890, "dur": 21.672,
    "args": {
      "External id": 2464480,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572940.681, "dur": 5.243,
    "args": {
      "External id": 2464481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572942.119, "dur": 3.213,
    "args": {
      "External id": 2464482,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572943.888, "dur": 1.219,
    "args": {
      "External id": 2464483,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572948.584, "dur": 75.253,
    "args": {
      "External id": 2464484,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656572949.850, "dur": 72.882,
    "args": {
      "External id": 2464485,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573029.588, "dur": 17.261,
    "args": {
      "External id": 2464486,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573052.052, "dur": 28.073,
    "args": {
      "External id": 2464487,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573056.280, "dur": 23.444,
    "args": {
      "External id": 2464488,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573063.666, "dur": 1.003,
    "args": {
      "External id": 2464489,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573086.764, "dur": 27.633,
    "args": {
      "External id": 2464490,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573088.394, "dur": 25.805,
    "args": {
      "External id": 2464491,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573093.361, "dur": 4.116,
    "args": {
      "External id": 2464492,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573098.507, "dur": 15.247,
    "args": {
      "External id": 2464493,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573130.419, "dur": 6.122,
    "args": {
      "External id": 2464494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573132.382, "dur": 3.912,
    "args": {
      "External id": 2464495,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573137.883, "dur": 3.271,
    "args": {
      "External id": 2464496,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573140.268, "dur": 0.798,
    "args": {
      "External id": 2464497,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573182.901, "dur": 22.403,
    "args": {
      "External id": 2464498,"Sequence number": 24740625, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9097
    }
  },
  {
    "ph": "s", "id": 23, "pid": 1336756, "tid": 1336756, "ts": 1514656573182.901,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573211.683, "dur": 7.142,
    "args": {
      "External id": 2464499,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573216.137, "dur": 1.285,
    "args": {
      "External id": 2464500,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573221.296, "dur": 6.369,
    "args": {
      "External id": 2464501,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "1"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573225.459, "dur": 1.052,
    "args": {
      "External id": 2464502,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573230.527, "dur": 2.662,
    "args": {
      "External id": 2464503,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573231.947, "dur": 0.655,
    "args": {
      "External id": 2464504,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573237.251, "dur": 5.455,
    "args": {
      "External id": 2464505,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9104
    }
  },
  {
    "ph": "s", "id": 22, "pid": 1336756, "tid": 1336756, "ts": 1514656573237.251,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573240.534, "dur": 0.897,
    "args": {
      "External id": 2464506,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573243.927, "dur": 6.386,
    "args": {
      "External id": 2464507,"Sequence number": 24740627, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9106
    }
  },
  {
    "ph": "s", "id": 21, "pid": 1336756, "tid": 1336756, "ts": 1514656573243.927,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573248.830, "dur": 0.682,
    "args": {
      "External id": 2464508,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573251.424, "dur": 4.893,
    "args": {
      "External id": 2464509,"Sequence number": 24740628, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9108
    }
  },
  {
    "ph": "s", "id": 20, "pid": 1336756, "tid": 1336756, "ts": 1514656573251.424,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573254.893, "dur": 0.618,
    "args": {
      "External id": 2464510,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573257.280, "dur": 4.820,
    "args": {
      "External id": 2464511,"Sequence number": 24740629, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9110
    }
  },
  {
    "ph": "s", "id": 19, "pid": 1336756, "tid": 1336756, "ts": 1514656573257.280,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573260.139, "dur": 1.173,
    "args": {
      "External id": 2464512,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573265.499, "dur": 28.423,
    "args": {
      "External id": 2464513,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573266.887, "dur": 26.800,
    "args": {
      "External id": 2464514,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573269.313, "dur": 7.296,
    "args": {
      "External id": 2464515,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573272.914, "dur": 3.059,
    "args": {
      "External id": 2464516,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573277.630, "dur": 15.605,
    "args": {
      "External id": 2464517,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573320.214, "dur": 3.970,
    "args": {
      "External id": 2464518,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9117
    }
  },
  {
    "ph": "s", "id": 18, "pid": 1336756, "tid": 1336756, "ts": 1514656573320.214,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573326.388, "dur": 1.363,
    "args": {
      "External id": 2464519,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573359.472, "dur": 42048.258,
    "args": {
      "External id": 2464520,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9119
    }
  },
  {
    "ph": "s", "id": 17, "pid": 1336756, "tid": 1336756, "ts": 1514656573359.472,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573373.403, "dur": 27.010,
    "args": {
      "External id": 2464521,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573374.382, "dur": 25.810,
    "args": {
      "External id": 2464522,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573375.585, "dur": 8.791,
    "args": {
      "External id": 2464523,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573378.590, "dur": 5.307,
    "args": {
      "External id": 2464524,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573385.004, "dur": 14.651,
    "args": {
      "External id": 2464525,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573415.738, "dur": 23.533,
    "args": {
      "External id": 2464526,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573417.123, "dur": 5.159,
    "args": {
      "External id": 2464527,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573419.083, "dur": 2.898,
    "args": {
      "External id": 2464528,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573423.614, "dur": 15.401,
    "args": {
      "External id": 2464529,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573425.032, "dur": 13.496,
    "args": {
      "External id": 2464530,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573442.768, "dur": 19.268,
    "args": {
      "External id": 2464531,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573443.284, "dur": 4.261,
    "args": {
      "External id": 2464532,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573444.545, "dur": 2.703,
    "args": {
      "External id": 2464533,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573447.999, "dur": 13.849,
    "args": {
      "External id": 2464534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573450.393, "dur": 11.147,
    "args": {
      "External id": 2464535,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573467.523, "dur": 16.116,
    "args": {
      "External id": 2464536,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573469.127, "dur": 2.722,
    "args": {
      "External id": 2464537,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573472.476, "dur": 10.903,
    "args": {
      "External id": 2464538,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573473.312, "dur": 9.774,
    "args": {
      "External id": 2464539,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573488.471, "dur": 19.783,
    "args": {
      "External id": 2464540,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573510.711, "dur": 67.095,
    "args": {
      "External id": 2464541,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573512.311, "dur": 65.127,
    "args": {
      "External id": 2464542,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573517.914, "dur": 0.885,
    "args": {
      "External id": 2464543,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573519.907, "dur": 42.533,
    "args": {
      "External id": 2464544,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573523.260, "dur": 38.852,
    "args": {
      "External id": 2464545,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573525.522, "dur": 19.681,
    "args": {
      "External id": 2464546,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573547.154, "dur": 14.592,
    "args": {
      "External id": 2464547,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573581.967, "dur": 36398.558,
    "args": {
      "External id": 2464548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1336756,
    "ts": 1514656573583.694, "dur": 36396.041,
    "args": {
      "External id": 2464549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610025.442, "dur": 12.887,
    "args": {
      "External id": 2464550,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610035.386, "dur": 1.137,
    "args": {
      "External id": 2464551,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610045.296, "dur": 100.994,
    "args": {
      "External id": 2464552,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610046.693, "dur": 5.979,
    "args": {
      "External id": 2464553,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610048.731, "dur": 2.864,
    "args": {
      "External id": 2464554,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610050.335, "dur": 0.980,
    "args": {
      "External id": 2464555,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610053.966, "dur": 91.555,
    "args": {
      "External id": 2464556,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610055.179, "dur": 89.468,
    "args": {
      "External id": 2464557,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610150.338, "dur": 4.176,
    "args": {
      "External id": 2464558,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610152.615, "dur": 0.576,
    "args": {
      "External id": 2464559,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610162.737, "dur": 2.303,
    "args": {
      "External id": 2464560,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610173.633, "dur": 5.925,
    "args": {
      "External id": 2464561,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610175.466, "dur": 3.816,
    "args": {
      "External id": 2464562,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610298.229, "dur": 191.959,
    "args": {
      "External id": 2464563,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610302.277, "dur": 2.272,
    "args": {
      "External id": 2464564,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610307.539, "dur": 182.171,
    "args": {
      "External id": 2464565,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610309.006, "dur": 0.325,
    "args": {
      "External id": 2464566,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610310.832, "dur": 24.268,
    "args": {
      "External id": 2464567,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610336.938, "dur": 4.974,
    "args": {
      "External id": 2464568,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610340.572, "dur": 1.019,
    "args": {
      "External id": 2464569,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610342.817, "dur": 22.390,
    "args": {
      "External id": 2464570,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610343.773, "dur": 1.533,
    "args": {
      "External id": 2464571,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610346.649, "dur": 18.209,
    "args": {
      "External id": 2464572,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610350.257, "dur": 3.047,
    "args": {
      "External id": 2464573,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610368.558, "dur": 22.835,
    "args": {
      "External id": 2464574,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610393.195, "dur": 14.485,
    "args": {
      "External id": 2464575,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610410.452, "dur": 13.589,
    "args": {
      "External id": 2464576,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610425.647, "dur": 12.958,
    "args": {
      "External id": 2464577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610440.378, "dur": 17.890,
    "args": {
      "External id": 2464578,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610441.992, "dur": 1.441,
    "args": {
      "External id": 2464579,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610445.346, "dur": 0.576,
    "args": {
      "External id": 2464580,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610461.795, "dur": 12.190,
    "args": {
      "External id": 2464581,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610475.288, "dur": 13.109,
    "args": {
      "External id": 2464582,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610496.328, "dur": 1.960,
    "args": {
      "External id": 2464583,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610504.327, "dur": 3.300,
    "args": {
      "External id": 2464584,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610506.284, "dur": 0.467,
    "args": {
      "External id": 2464585,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610589.946, "dur": 65.092,
    "args": {
      "External id": 2464586,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610660.782, "dur": 8.116,
    "args": {
      "External id": 2464587,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610663.833, "dur": 0.915,
    "args": {
      "External id": 2464588,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610672.542, "dur": 31.751,
    "args": {
      "External id": 2464589,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610710.380, "dur": 6.318,
    "args": {
      "External id": 2464590,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610712.075, "dur": 3.671,
    "args": {
      "External id": 2464591,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610714.218, "dur": 1.295,
    "args": {
      "External id": 2464592,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610719.976, "dur": 42.162,
    "args": {
      "External id": 2464593,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610721.597, "dur": 39.940,
    "args": {
      "External id": 2464594,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610766.132, "dur": 14.180,
    "args": {
      "External id": 2464595,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610788.182, "dur": 3.610,
    "args": {
      "External id": 2464596,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610790.149, "dur": 0.675,
    "args": {
      "External id": 2464597,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610796.033, "dur": 49.981,
    "args": {
      "External id": 2464598,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610797.223, "dur": 3.553,
    "args": {
      "External id": 2464599,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610797.907, "dur": 2.288,
    "args": {
      "External id": 2464600,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610799.172, "dur": 0.888,
    "args": {
      "External id": 2464601,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610803.306, "dur": 42.200,
    "args": {
      "External id": 2464602,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610804.095, "dur": 40.806,
    "args": {
      "External id": 2464603,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610849.855, "dur": 5.431,
    "args": {
      "External id": 2464604,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610851.716, "dur": 2.421,
    "args": {
      "External id": 2464605,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610861.402, "dur": 1.591,
    "args": {
      "External id": 2464606,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610870.476, "dur": 7.982,
    "args": {
      "External id": 2464607,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610874.368, "dur": 3.844,
    "args": {
      "External id": 2464608,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610960.221, "dur": 225.009,
    "args": {
      "External id": 2464609,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610966.469, "dur": 2.222,
    "args": {
      "External id": 2464610,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610971.672, "dur": 213.113,
    "args": {
      "External id": 2464611,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610972.776, "dur": 0.384,
    "args": {
      "External id": 2464612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656610974.685, "dur": 56.353,
    "args": {
      "External id": 2464613,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611033.637, "dur": 11.091,
    "args": {
      "External id": 2464614,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611037.824, "dur": 6.605,
    "args": {
      "External id": 2464615,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611047.809, "dur": 22.560,
    "args": {
      "External id": 2464616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611048.999, "dur": 2.134,
    "args": {
      "External id": 2464617,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611052.433, "dur": 17.676,
    "args": {
      "External id": 2464618,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611055.648, "dur": 2.718,
    "args": {
      "External id": 2464619,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611071.994, "dur": 20.652,
    "args": {
      "External id": 2464620,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611094.095, "dur": 13.002,
    "args": {
      "External id": 2464621,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611109.993, "dur": 12.663,
    "args": {
      "External id": 2464622,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611124.153, "dur": 11.914,
    "args": {
      "External id": 2464623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611137.675, "dur": 21.608,
    "args": {
      "External id": 2464624,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611140.934, "dur": 1.057,
    "args": {
      "External id": 2464625,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611144.136, "dur": 2.053,
    "args": {
      "External id": 2464626,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611160.490, "dur": 11.356,
    "args": {
      "External id": 2464627,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611172.850, "dur": 10.939,
    "args": {
      "External id": 2464628,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611192.599, "dur": 2.029,
    "args": {
      "External id": 2464629,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611204.082, "dur": 3.697,
    "args": {
      "External id": 2464630,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611206.337, "dur": 0.557,
    "args": {
      "External id": 2464631,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611274.187, "dur": 54.463,
    "args": {
      "External id": 2464632,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611335.149, "dur": 4.141,
    "args": {
      "External id": 2464633,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611337.412, "dur": 0.879,
    "args": {
      "External id": 2464634,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611340.467, "dur": 22.963,
    "args": {
      "External id": 2464635,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611368.121, "dur": 5.490,
    "args": {
      "External id": 2464636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611369.332, "dur": 3.635,
    "args": {
      "External id": 2464637,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611370.982, "dur": 1.821,
    "args": {
      "External id": 2464638,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611377.807, "dur": 41.476,
    "args": {
      "External id": 2464639,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611379.193, "dur": 39.561,
    "args": {
      "External id": 2464640,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611422.922, "dur": 13.636,
    "args": {
      "External id": 2464641,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611442.172, "dur": 3.472,
    "args": {
      "External id": 2464642,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611444.018, "dur": 0.842,
    "args": {
      "External id": 2464643,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611449.400, "dur": 49.071,
    "args": {
      "External id": 2464644,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611450.433, "dur": 5.384,
    "args": {
      "External id": 2464645,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611453.144, "dur": 2.116,
    "args": {
      "External id": 2464646,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611454.586, "dur": 0.543,
    "args": {
      "External id": 2464647,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611456.980, "dur": 41.081,
    "args": {
      "External id": 2464648,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611457.630, "dur": 39.883,
    "args": {
      "External id": 2464649,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611502.714, "dur": 3.138,
    "args": {
      "External id": 2464650,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611504.293, "dur": 0.491,
    "args": {
      "External id": 2464651,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611510.687, "dur": 1.658,
    "args": {
      "External id": 2464652,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611520.991, "dur": 5.610,
    "args": {
      "External id": 2464653,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611522.896, "dur": 3.417,
    "args": {
      "External id": 2464654,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611632.721, "dur": 180.371,
    "args": {
      "External id": 2464655,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611637.087, "dur": 2.888,
    "args": {
      "External id": 2464656,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611641.307, "dur": 171.376,
    "args": {
      "External id": 2464657,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611642.823, "dur": 0.327,
    "args": {
      "External id": 2464658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611644.152, "dur": 20.552,
    "args": {
      "External id": 2464659,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611666.377, "dur": 6.085,
    "args": {
      "External id": 2464660,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611670.505, "dur": 1.475,
    "args": {
      "External id": 2464661,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611675.040, "dur": 21.001,
    "args": {
      "External id": 2464662,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611676.041, "dur": 1.455,
    "args": {
      "External id": 2464663,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611678.876, "dur": 16.918,
    "args": {
      "External id": 2464664,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611682.185, "dur": 2.743,
    "args": {
      "External id": 2464665,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611697.309, "dur": 22.642,
    "args": {
      "External id": 2464666,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611721.216, "dur": 14.607,
    "args": {
      "External id": 2464667,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611738.991, "dur": 12.776,
    "args": {
      "External id": 2464668,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611753.248, "dur": 11.539,
    "args": {
      "External id": 2464669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611766.272, "dur": 20.919,
    "args": {
      "External id": 2464670,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611769.907, "dur": 1.335,
    "args": {
      "External id": 2464671,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611772.957, "dur": 1.163,
    "args": {
      "External id": 2464672,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611788.402, "dur": 11.178,
    "args": {
      "External id": 2464673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611800.938, "dur": 10.763,
    "args": {
      "External id": 2464674,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611819.102, "dur": 2.173,
    "args": {
      "External id": 2464675,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611829.890, "dur": 3.665,
    "args": {
      "External id": 2464676,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611832.153, "dur": 0.618,
    "args": {
      "External id": 2464677,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611900.160, "dur": 52.732,
    "args": {
      "External id": 2464678,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611958.025, "dur": 4.438,
    "args": {
      "External id": 2464679,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611960.372, "dur": 1.120,
    "args": {
      "External id": 2464680,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656611963.778, "dur": 62.260,
    "args": {
      "External id": 2464681,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612032.545, "dur": 7.813,
    "args": {
      "External id": 2464682,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612034.028, "dur": 5.554,
    "args": {
      "External id": 2464683,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612037.718, "dur": 1.656,
    "args": {
      "External id": 2464684,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612043.434, "dur": 75.544,
    "args": {
      "External id": 2464685,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612044.522, "dur": 73.940,
    "args": {
      "External id": 2464686,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612122.673, "dur": 35.769,
    "args": {
      "External id": 2464687,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612164.504, "dur": 4.023,
    "args": {
      "External id": 2464688,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612166.882, "dur": 0.826,
    "args": {
      "External id": 2464689,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612172.708, "dur": 59.568,
    "args": {
      "External id": 2464690,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612175.115, "dur": 3.539,
    "args": {
      "External id": 2464691,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612176.010, "dur": 2.088,
    "args": {
      "External id": 2464692,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612177.304, "dur": 0.640,
    "args": {
      "External id": 2464693,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612179.334, "dur": 52.394,
    "args": {
      "External id": 2464694,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612179.817, "dur": 51.388,
    "args": {
      "External id": 2464695,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612235.915, "dur": 3.389,
    "args": {
      "External id": 2464696,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612237.508, "dur": 0.678,
    "args": {
      "External id": 2464697,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612246.624, "dur": 1.367,
    "args": {
      "External id": 2464698,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612255.055, "dur": 7.241,
    "args": {
      "External id": 2464699,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612256.828, "dur": 5.166,
    "args": {
      "External id": 2464700,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612346.588, "dur": 167.154,
    "args": {
      "External id": 2464701,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612348.766, "dur": 1.926,
    "args": {
      "External id": 2464702,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612353.719, "dur": 159.476,
    "args": {
      "External id": 2464703,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612354.812, "dur": 0.267,
    "args": {
      "External id": 2464704,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612356.231, "dur": 20.464,
    "args": {
      "External id": 2464705,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612378.019, "dur": 3.502,
    "args": {
      "External id": 2464706,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612380.099, "dur": 1.139,
    "args": {
      "External id": 2464707,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612382.430, "dur": 20.845,
    "args": {
      "External id": 2464708,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612383.301, "dur": 1.489,
    "args": {
      "External id": 2464709,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612386.034, "dur": 16.942,
    "args": {
      "External id": 2464710,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612390.178, "dur": 2.412,
    "args": {
      "External id": 2464711,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612406.564, "dur": 19.945,
    "args": {
      "External id": 2464712,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612427.820, "dur": 11.989,
    "args": {
      "External id": 2464713,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612442.574, "dur": 12.543,
    "args": {
      "External id": 2464714,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612456.335, "dur": 11.356,
    "args": {
      "External id": 2464715,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612469.233, "dur": 18.073,
    "args": {
      "External id": 2464716,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612471.035, "dur": 1.402,
    "args": {
      "External id": 2464717,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612474.102, "dur": 0.883,
    "args": {
      "External id": 2464718,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612488.565, "dur": 11.058,
    "args": {
      "External id": 2464719,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612502.535, "dur": 9.800,
    "args": {
      "External id": 2464720,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612519.276, "dur": 1.588,
    "args": {
      "External id": 2464721,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612544.477, "dur": 4.230,
    "args": {
      "External id": 2464722,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612547.012, "dur": 0.544,
    "args": {
      "External id": 2464723,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612613.392, "dur": 52.249,
    "args": {
      "External id": 2464724,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612670.881, "dur": 4.389,
    "args": {
      "External id": 2464725,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612673.177, "dur": 1.077,
    "args": {
      "External id": 2464726,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612676.835, "dur": 24.573,
    "args": {
      "External id": 2464727,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612707.558, "dur": 4.912,
    "args": {
      "External id": 2464728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612708.940, "dur": 2.949,
    "args": {
      "External id": 2464729,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612710.802, "dur": 0.892,
    "args": {
      "External id": 2464730,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612714.719, "dur": 39.075,
    "args": {
      "External id": 2464731,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612715.988, "dur": 37.169,
    "args": {
      "External id": 2464732,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612757.218, "dur": 14.093,
    "args": {
      "External id": 2464733,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612776.473, "dur": 5.382,
    "args": {
      "External id": 2464734,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612780.339, "dur": 0.845,
    "args": {
      "External id": 2464735,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612785.458, "dur": 51.092,
    "args": {
      "External id": 2464736,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612786.326, "dur": 3.270,
    "args": {
      "External id": 2464737,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612787.183, "dur": 1.933,
    "args": {
      "External id": 2464738,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612788.318, "dur": 0.648,
    "args": {
      "External id": 2464739,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612790.303, "dur": 45.828,
    "args": {
      "External id": 2464740,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612792.715, "dur": 42.909,
    "args": {
      "External id": 2464741,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612840.520, "dur": 5.178,
    "args": {
      "External id": 2464742,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612842.464, "dur": 2.114,
    "args": {
      "External id": 2464743,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612850.986, "dur": 1.449,
    "args": {
      "External id": 2464744,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612859.182, "dur": 6.338,
    "args": {
      "External id": 2464745,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612860.991, "dur": 4.282,
    "args": {
      "External id": 2464746,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612942.012, "dur": 206.259,
    "args": {
      "External id": 2464747,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612944.123, "dur": 1.980,
    "args": {
      "External id": 2464748,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612948.121, "dur": 199.630,
    "args": {
      "External id": 2464749,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612949.563, "dur": 0.253,
    "args": {
      "External id": 2464750,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612950.846, "dur": 20.641,
    "args": {
      "External id": 2464751,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612973.052, "dur": 3.127,
    "args": {
      "External id": 2464752,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612974.983, "dur": 0.967,
    "args": {
      "External id": 2464753,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612977.104, "dur": 60.418,
    "args": {
      "External id": 2464754,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656612978.387, "dur": 1.483,
    "args": {
      "External id": 2464755,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613013.887, "dur": 23.216,
    "args": {
      "External id": 2464756,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613020.732, "dur": 3.154,
    "args": {
      "External id": 2464757,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613038.875, "dur": 19.172,
    "args": {
      "External id": 2464758,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613059.719, "dur": 12.665,
    "args": {
      "External id": 2464759,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613074.717, "dur": 13.152,
    "args": {
      "External id": 2464760,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613089.313, "dur": 11.818,
    "args": {
      "External id": 2464761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613102.822, "dur": 17.998,
    "args": {
      "External id": 2464762,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613104.621, "dur": 1.214,
    "args": {
      "External id": 2464763,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613107.334, "dur": 1.041,
    "args": {
      "External id": 2464764,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613123.884, "dur": 11.045,
    "args": {
      "External id": 2464765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613136.349, "dur": 10.511,
    "args": {
      "External id": 2464766,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613155.499, "dur": 2.238,
    "args": {
      "External id": 2464767,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613166.784, "dur": 38.142,
    "args": {
      "External id": 2464768,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613203.814, "dur": 0.352,
    "args": {
      "External id": 2464769,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613272.912, "dur": 54.163,
    "args": {
      "External id": 2464770,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613331.668, "dur": 5.579,
    "args": {
      "External id": 2464771,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613334.144, "dur": 2.088,
    "args": {
      "External id": 2464772,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613340.424, "dur": 22.754,
    "args": {
      "External id": 2464773,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613367.404, "dur": 4.752,
    "args": {
      "External id": 2464774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613368.674, "dur": 2.855,
    "args": {
      "External id": 2464775,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613370.248, "dur": 1.088,
    "args": {
      "External id": 2464776,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613374.562, "dur": 39.154,
    "args": {
      "External id": 2464777,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613375.842, "dur": 37.345,
    "args": {
      "External id": 2464778,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613417.476, "dur": 13.317,
    "args": {
      "External id": 2464779,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613437.621, "dur": 3.511,
    "args": {
      "External id": 2464780,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613439.554, "dur": 0.868,
    "args": {
      "External id": 2464781,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613444.619, "dur": 46.196,
    "args": {
      "External id": 2464782,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613445.345, "dur": 3.013,
    "args": {
      "External id": 2464783,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613445.943, "dur": 1.844,
    "args": {
      "External id": 2464784,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613446.937, "dur": 0.719,
    "args": {
      "External id": 2464785,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613450.443, "dur": 39.895,
    "args": {
      "External id": 2464786,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613450.914, "dur": 38.930,
    "args": {
      "External id": 2464787,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613494.291, "dur": 3.624,
    "args": {
      "External id": 2464788,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613495.986, "dur": 0.964,
    "args": {
      "External id": 2464789,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613502.606, "dur": 1.316,
    "args": {
      "External id": 2464790,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613510.400, "dur": 7.715,
    "args": {
      "External id": 2464791,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613513.687, "dur": 4.138,
    "args": {
      "External id": 2464792,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613615.430, "dur": 174.559,
    "args": {
      "External id": 2464793,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613617.871, "dur": 4.027,
    "args": {
      "External id": 2464794,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613623.570, "dur": 165.901,
    "args": {
      "External id": 2464795,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613624.720, "dur": 0.286,
    "args": {
      "External id": 2464796,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613625.982, "dur": 20.441,
    "args": {
      "External id": 2464797,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613647.863, "dur": 4.621,
    "args": {
      "External id": 2464798,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613651.298, "dur": 0.921,
    "args": {
      "External id": 2464799,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613653.462, "dur": 24.975,
    "args": {
      "External id": 2464800,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613656.234, "dur": 1.793,
    "args": {
      "External id": 2464801,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613659.215, "dur": 18.971,
    "args": {
      "External id": 2464802,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613661.648, "dur": 2.545,
    "args": {
      "External id": 2464803,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613679.684, "dur": 20.812,
    "args": {
      "External id": 2464804,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613701.733, "dur": 11.801,
    "args": {
      "External id": 2464805,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613715.978, "dur": 12.551,
    "args": {
      "External id": 2464806,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613729.711, "dur": 11.636,
    "args": {
      "External id": 2464807,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613742.811, "dur": 20.607,
    "args": {
      "External id": 2464808,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613746.194, "dur": 1.224,
    "args": {
      "External id": 2464809,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613750.737, "dur": 0.693,
    "args": {
      "External id": 2464810,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613764.780, "dur": 10.929,
    "args": {
      "External id": 2464811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613776.595, "dur": 11.868,
    "args": {
      "External id": 2464812,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613796.291, "dur": 1.864,
    "args": {
      "External id": 2464813,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613806.776, "dur": 3.312,
    "args": {
      "External id": 2464814,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613808.849, "dur": 0.427,
    "args": {
      "External id": 2464815,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613866.910, "dur": 50.128,
    "args": {
      "External id": 2464816,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613923.225, "dur": 4.101,
    "args": {
      "External id": 2464817,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613925.497, "dur": 0.924,
    "args": {
      "External id": 2464818,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613928.590, "dur": 22.220,
    "args": {
      "External id": 2464819,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613955.237, "dur": 4.681,
    "args": {
      "External id": 2464820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613956.500, "dur": 2.874,
    "args": {
      "External id": 2464821,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613957.875, "dur": 1.325,
    "args": {
      "External id": 2464822,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613963.529, "dur": 75.701,
    "args": {
      "External id": 2464823,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656613964.676, "dur": 73.445,
    "args": {
      "External id": 2464824,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614044.553, "dur": 15.183,
    "args": {
      "External id": 2464825,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614065.680, "dur": 4.079,
    "args": {
      "External id": 2464826,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614067.976, "dur": 0.811,
    "args": {
      "External id": 2464827,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614073.525, "dur": 50.527,
    "args": {
      "External id": 2464828,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614074.415, "dur": 4.957,
    "args": {
      "External id": 2464829,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614076.783, "dur": 2.051,
    "args": {
      "External id": 2464830,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614077.796, "dur": 0.922,
    "args": {
      "External id": 2464831,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614080.137, "dur": 43.432,
    "args": {
      "External id": 2464832,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614081.056, "dur": 41.862,
    "args": {
      "External id": 2464833,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614128.173, "dur": 3.685,
    "args": {
      "External id": 2464834,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614130.004, "dur": 0.828,
    "args": {
      "External id": 2464835,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614137.452, "dur": 1.312,
    "args": {
      "External id": 2464836,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614147.054, "dur": 5.378,
    "args": {
      "External id": 2464837,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614148.543, "dur": 3.641,
    "args": {
      "External id": 2464838,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614232.091, "dur": 173.247,
    "args": {
      "External id": 2464839,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614236.145, "dur": 2.242,
    "args": {
      "External id": 2464840,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614239.926, "dur": 164.834,
    "args": {
      "External id": 2464841,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614240.861, "dur": 0.416,
    "args": {
      "External id": 2464842,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614242.178, "dur": 19.391,
    "args": {
      "External id": 2464843,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614264.681, "dur": 4.495,
    "args": {
      "External id": 2464844,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614268.199, "dur": 0.693,
    "args": {
      "External id": 2464845,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614270.215, "dur": 19.391,
    "args": {
      "External id": 2464846,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614271.056, "dur": 1.457,
    "args": {
      "External id": 2464847,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614273.508, "dur": 15.805,
    "args": {
      "External id": 2464848,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614275.752, "dur": 2.358,
    "args": {
      "External id": 2464849,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614290.822, "dur": 18.501,
    "args": {
      "External id": 2464850,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614310.455, "dur": 14.780,
    "args": {
      "External id": 2464851,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614327.551, "dur": 12.433,
    "args": {
      "External id": 2464852,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614341.144, "dur": 11.621,
    "args": {
      "External id": 2464853,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614356.170, "dur": 23.176,
    "args": {
      "External id": 2464854,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614357.729, "dur": 1.130,
    "args": {
      "External id": 2464855,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614366.660, "dur": 0.801,
    "args": {
      "External id": 2464856,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614380.740, "dur": 11.001,
    "args": {
      "External id": 2464857,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614392.846, "dur": 10.926,
    "args": {
      "External id": 2464858,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614411.265, "dur": 1.523,
    "args": {
      "External id": 2464859,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614419.850, "dur": 3.293,
    "args": {
      "External id": 2464860,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614421.823, "dur": 0.503,
    "args": {
      "External id": 2464861,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614482.654, "dur": 66.067,
    "args": {
      "External id": 2464862,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614555.197, "dur": 5.424,
    "args": {
      "External id": 2464863,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614558.224, "dur": 1.069,
    "args": {
      "External id": 2464864,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614562.141, "dur": 24.145,
    "args": {
      "External id": 2464865,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614593.873, "dur": 11.702,
    "args": {
      "External id": 2464866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614595.157, "dur": 9.798,
    "args": {
      "External id": 2464867,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614602.102, "dur": 2.605,
    "args": {
      "External id": 2464868,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614608.728, "dur": 44.172,
    "args": {
      "External id": 2464869,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614609.942, "dur": 42.129,
    "args": {
      "External id": 2464870,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614656.698, "dur": 14.272,
    "args": {
      "External id": 2464871,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614676.921, "dur": 3.516,
    "args": {
      "External id": 2464872,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614679.065, "dur": 0.702,
    "args": {
      "External id": 2464873,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614685.965, "dur": 46.014,
    "args": {
      "External id": 2464874,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614686.676, "dur": 3.165,
    "args": {
      "External id": 2464875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614687.290, "dur": 1.911,
    "args": {
      "External id": 2464876,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614688.466, "dur": 0.606,
    "args": {
      "External id": 2464877,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614690.337, "dur": 41.203,
    "args": {
      "External id": 2464878,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614690.941, "dur": 39.887,
    "args": {
      "External id": 2464879,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614737.389, "dur": 3.635,
    "args": {
      "External id": 2464880,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614739.103, "dur": 0.874,
    "args": {
      "External id": 2464881,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614746.778, "dur": 1.425,
    "args": {
      "External id": 2464882,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614755.741, "dur": 6.140,
    "args": {
      "External id": 2464883,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614757.543, "dur": 4.085,
    "args": {
      "External id": 2464884,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614844.097, "dur": 207.547,
    "args": {
      "External id": 2464885,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614846.040, "dur": 2.064,
    "args": {
      "External id": 2464886,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614851.439, "dur": 199.734,
    "args": {
      "External id": 2464887,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614852.486, "dur": 0.427,
    "args": {
      "External id": 2464888,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614853.764, "dur": 20.283,
    "args": {
      "External id": 2464889,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614875.464, "dur": 4.470,
    "args": {
      "External id": 2464890,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614877.444, "dur": 2.057,
    "args": {
      "External id": 2464891,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614880.865, "dur": 22.056,
    "args": {
      "External id": 2464892,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614882.077, "dur": 1.783,
    "args": {
      "External id": 2464893,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614885.202, "dur": 17.465,
    "args": {
      "External id": 2464894,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614889.453, "dur": 2.702,
    "args": {
      "External id": 2464895,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614904.365, "dur": 18.732,
    "args": {
      "External id": 2464896,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614924.264, "dur": 12.008,
    "args": {
      "External id": 2464897,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614939.037, "dur": 12.662,
    "args": {
      "External id": 2464898,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614952.886, "dur": 11.618,
    "args": {
      "External id": 2464899,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614965.900, "dur": 53.406,
    "args": {
      "External id": 2464900,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614967.521, "dur": 1.090,
    "args": {
      "External id": 2464901,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656614970.364, "dur": 1.753,
    "args": {
      "External id": 2464902,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615023.809, "dur": 13.668,
    "args": {
      "External id": 2464903,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615038.654, "dur": 11.305,
    "args": {
      "External id": 2464904,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615059.365, "dur": 2.431,
    "args": {
      "External id": 2464905,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615070.334, "dur": 3.369,
    "args": {
      "External id": 2464906,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615072.344, "dur": 0.499,
    "args": {
      "External id": 2464907,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615137.082, "dur": 62.852,
    "args": {
      "External id": 2464908,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615204.784, "dur": 4.485,
    "args": {
      "External id": 2464909,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615207.506, "dur": 0.761,
    "args": {
      "External id": 2464910,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615212.164, "dur": 25.368,
    "args": {
      "External id": 2464911,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615241.962, "dur": 5.120,
    "args": {
      "External id": 2464912,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615243.338, "dur": 3.162,
    "args": {
      "External id": 2464913,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615244.772, "dur": 1.555,
    "args": {
      "External id": 2464914,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615249.362, "dur": 40.655,
    "args": {
      "External id": 2464915,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615250.316, "dur": 39.086,
    "args": {
      "External id": 2464916,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615293.606, "dur": 13.702,
    "args": {
      "External id": 2464917,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615313.322, "dur": 21.321,
    "args": {
      "External id": 2464918,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615315.497, "dur": 18.774,
    "args": {
      "External id": 2464919,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615320.182, "dur": 0.762,
    "args": {
      "External id": 2464920,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615339.054, "dur": 23.977,
    "args": {
      "External id": 2464921,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615340.865, "dur": 21.963,
    "args": {
      "External id": 2464922,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615345.246, "dur": 3.660,
    "args": {
      "External id": 2464923,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615349.975, "dur": 12.287,
    "args": {
      "External id": 2464924,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615376.616, "dur": 4.646,
    "args": {
      "External id": 2464925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615378.415, "dur": 2.577,
    "args": {
      "External id": 2464926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615382.332, "dur": 2.620,
    "args": {
      "External id": 2464927,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615384.268, "dur": 0.600,
    "args": {
      "External id": 2464928,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615423.168, "dur": 21.193,
    "args": {
      "External id": 2464929,"Sequence number": 24740632, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615445.903, "dur": 13.038,
    "args": {
      "External id": 2464930,"Sequence number": 24740633, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9529
    }
  },
  {
    "ph": "s", "id": 16, "pid": 1336756, "tid": 1336756, "ts": 1514656615445.903,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615464.467, "dur": 5.223,
    "args": {
      "External id": 2464931,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615467.454, "dur": 0.892,
    "args": {
      "External id": 2464932,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615472.016, "dur": 7.790,
    "args": {
      "External id": 2464933,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615477.699, "dur": 0.814,
    "args": {
      "External id": 2464934,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615481.028, "dur": 2.888,
    "args": {
      "External id": 2464935,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615482.523, "dur": 0.827,
    "args": {
      "External id": 2464936,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615487.653, "dur": 5.661,
    "args": {
      "External id": 2464937,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9536
    }
  },
  {
    "ph": "s", "id": 15, "pid": 1336756, "tid": 1336756, "ts": 1514656615487.653,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615490.912, "dur": 1.110,
    "args": {
      "External id": 2464938,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615494.492, "dur": 5.977,
    "args": {
      "External id": 2464939,"Sequence number": 24740635, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9538
    }
  },
  {
    "ph": "s", "id": 14, "pid": 1336756, "tid": 1336756, "ts": 1514656615494.492,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615499.032, "dur": 0.724,
    "args": {
      "External id": 2464940,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615501.478, "dur": 4.754,
    "args": {
      "External id": 2464941,"Sequence number": 24740636, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9540
    }
  },
  {
    "ph": "s", "id": 13, "pid": 1336756, "tid": 1336756, "ts": 1514656615501.478,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615504.520, "dur": 0.920,
    "args": {
      "External id": 2464942,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615507.156, "dur": 4.267,
    "args": {
      "External id": 2464943,"Sequence number": 24740637, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9542
    }
  },
  {
    "ph": "s", "id": 12, "pid": 1336756, "tid": 1336756, "ts": 1514656615507.156,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615509.772, "dur": 0.945,
    "args": {
      "External id": 2464944,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615515.420, "dur": 52.354,
    "args": {
      "External id": 2464945,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615517.053, "dur": 50.516,
    "args": {
      "External id": 2464946,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615519.510, "dur": 22.966,
    "args": {
      "External id": 2464947,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615523.176, "dur": 2.975,
    "args": {
      "External id": 2464948,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615544.492, "dur": 22.576,
    "args": {
      "External id": 2464949,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615594.523, "dur": 6.162,
    "args": {
      "External id": 2464950,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9549
    }
  },
  {
    "ph": "s", "id": 11, "pid": 1336756, "tid": 1336756, "ts": 1514656615594.523,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615603.365, "dur": 1.426,
    "args": {
      "External id": 2464951,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615635.751, "dur": 43304.203,
    "args": {
      "External id": 2464952,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9551
    }
  },
  {
    "ph": "s", "id": 10, "pid": 1336756, "tid": 1336756, "ts": 1514656615635.751,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615650.777, "dur": 27.283,
    "args": {
      "External id": 2464953,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615653.595, "dur": 24.278,
    "args": {
      "External id": 2464954,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615655.239, "dur": 6.460,
    "args": {
      "External id": 2464955,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615656.809, "dur": 4.481,
    "args": {
      "External id": 2464956,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615662.624, "dur": 14.872,
    "args": {
      "External id": 2464957,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615693.286, "dur": 23.749,
    "args": {
      "External id": 2464958,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615694.355, "dur": 5.898,
    "args": {
      "External id": 2464959,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615696.057, "dur": 3.891,
    "args": {
      "External id": 2464960,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615701.555, "dur": 15.244,
    "args": {
      "External id": 2464961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615703.503, "dur": 12.858,
    "args": {
      "External id": 2464962,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615720.329, "dur": 19.037,
    "args": {
      "External id": 2464963,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615721.337, "dur": 4.024,
    "args": {
      "External id": 2464964,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615722.601, "dur": 2.425,
    "args": {
      "External id": 2464965,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615727.560, "dur": 11.619,
    "args": {
      "External id": 2464966,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615728.093, "dur": 10.710,
    "args": {
      "External id": 2464967,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615744.806, "dur": 16.348,
    "args": {
      "External id": 2464968,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615748.157, "dur": 2.411,
    "args": {
      "External id": 2464969,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615751.224, "dur": 9.679,
    "args": {
      "External id": 2464970,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615751.756, "dur": 8.849,
    "args": {
      "External id": 2464971,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615766.174, "dur": 20.847,
    "args": {
      "External id": 2464972,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615789.683, "dur": 45.386,
    "args": {
      "External id": 2464973,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615793.223, "dur": 41.444,
    "args": {
      "External id": 2464974,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615797.712, "dur": 0.839,
    "args": {
      "External id": 2464975,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615799.867, "dur": 20.891,
    "args": {
      "External id": 2464976,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615801.250, "dur": 19.285,
    "args": {
      "External id": 2464977,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615805.023, "dur": 2.734,
    "args": {
      "External id": 2464978,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615808.824, "dur": 11.274,
    "args": {
      "External id": 2464979,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615838.667, "dur": 37657.914,
    "args": {
      "External id": 2464980,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1336756,
    "ts": 1514656615839.992, "dur": 37655.536,
    "args": {
      "External id": 2464981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653507.984, "dur": 6.699,
    "args": {
      "External id": 2464982,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653511.769, "dur": 1.126,
    "args": {
      "External id": 2464983,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653521.515, "dur": 113.694,
    "args": {
      "External id": 2464984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653522.964, "dur": 17.400,
    "args": {
      "External id": 2464985,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653524.964, "dur": 14.273,
    "args": {
      "External id": 2464986,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653526.639, "dur": 11.833,
    "args": {
      "External id": 2464987,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653542.231, "dur": 92.288,
    "args": {
      "External id": 2464988,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653543.773, "dur": 90.088,
    "args": {
      "External id": 2464989,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653639.478, "dur": 4.832,
    "args": {
      "External id": 2464990,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653642.070, "dur": 0.863,
    "args": {
      "External id": 2464991,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653653.216, "dur": 2.418,
    "args": {
      "External id": 2464992,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653664.450, "dur": 6.503,
    "args": {
      "External id": 2464993,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653666.119, "dur": 4.544,
    "args": {
      "External id": 2464994,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653793.486, "dur": 224.182,
    "args": {
      "External id": 2464995,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653796.225, "dur": 2.421,
    "args": {
      "External id": 2464996,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653803.633, "dur": 213.523,
    "args": {
      "External id": 2464997,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653805.009, "dur": 0.520,
    "args": {
      "External id": 2464998,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653806.785, "dur": 24.269,
    "args": {
      "External id": 2464999,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653832.657, "dur": 4.820,
    "args": {
      "External id": 2465000,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653834.786, "dur": 2.374,
    "args": {
      "External id": 2465001,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653838.576, "dur": 23.214,
    "args": {
      "External id": 2465002,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653839.732, "dur": 1.615,
    "args": {
      "External id": 2465003,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653842.790, "dur": 18.682,
    "args": {
      "External id": 2465004,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653846.150, "dur": 3.479,
    "args": {
      "External id": 2465005,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653865.343, "dur": 22.710,
    "args": {
      "External id": 2465006,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653889.828, "dur": 13.697,
    "args": {
      "External id": 2465007,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653906.486, "dur": 15.192,
    "args": {
      "External id": 2465008,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653923.205, "dur": 12.209,
    "args": {
      "External id": 2465009,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653937.007, "dur": 19.058,
    "args": {
      "External id": 2465010,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653939.039, "dur": 1.607,
    "args": {
      "External id": 2465011,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653942.826, "dur": 0.715,
    "args": {
      "External id": 2465012,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653959.529, "dur": 11.700,
    "args": {
      "External id": 2465013,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656653972.585, "dur": 42.436,
    "args": {
      "External id": 2465014,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654026.458, "dur": 2.345,
    "args": {
      "External id": 2465015,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654035.836, "dur": 4.447,
    "args": {
      "External id": 2465016,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654038.369, "dur": 0.995,
    "args": {
      "External id": 2465017,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654115.890, "dur": 67.145,
    "args": {
      "External id": 2465018,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654188.454, "dur": 7.935,
    "args": {
      "External id": 2465019,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654191.218, "dur": 2.402,
    "args": {
      "External id": 2465020,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654199.647, "dur": 27.034,
    "args": {
      "External id": 2465021,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654232.271, "dur": 6.676,
    "args": {
      "External id": 2465022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654234.111, "dur": 3.981,
    "args": {
      "External id": 2465023,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654236.422, "dur": 1.372,
    "args": {
      "External id": 2465024,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654241.875, "dur": 43.880,
    "args": {
      "External id": 2465025,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654243.337, "dur": 41.615,
    "args": {
      "External id": 2465026,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654290.494, "dur": 15.907,
    "args": {
      "External id": 2465027,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654314.225, "dur": 3.443,
    "args": {
      "External id": 2465028,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654316.136, "dur": 0.687,
    "args": {
      "External id": 2465029,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654321.830, "dur": 50.131,
    "args": {
      "External id": 2465030,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654322.892, "dur": 4.027,
    "args": {
      "External id": 2465031,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654324.061, "dur": 2.271,
    "args": {
      "External id": 2465032,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654325.350, "dur": 0.813,
    "args": {
      "External id": 2465033,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654329.239, "dur": 42.325,
    "args": {
      "External id": 2465034,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654329.775, "dur": 41.123,
    "args": {
      "External id": 2465035,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654375.882, "dur": 4.019,
    "args": {
      "External id": 2465036,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654377.944, "dur": 0.866,
    "args": {
      "External id": 2465037,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654385.324, "dur": 1.406,
    "args": {
      "External id": 2465038,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654394.167, "dur": 8.536,
    "args": {
      "External id": 2465039,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654398.137, "dur": 4.293,
    "args": {
      "External id": 2465040,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654491.520, "dur": 197.347,
    "args": {
      "External id": 2465041,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654493.963, "dur": 3.259,
    "args": {
      "External id": 2465042,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654498.527, "dur": 189.839,
    "args": {
      "External id": 2465043,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654499.709, "dur": 0.449,
    "args": {
      "External id": 2465044,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654501.949, "dur": 20.304,
    "args": {
      "External id": 2465045,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654523.639, "dur": 19.911,
    "args": {
      "External id": 2465046,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654541.861, "dur": 0.969,
    "args": {
      "External id": 2465047,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654546.626, "dur": 24.720,
    "args": {
      "External id": 2465048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654547.789, "dur": 2.401,
    "args": {
      "External id": 2465049,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654551.510, "dur": 19.593,
    "args": {
      "External id": 2465050,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654555.055, "dur": 3.293,
    "args": {
      "External id": 2465051,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654572.690, "dur": 20.630,
    "args": {
      "External id": 2465052,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654594.557, "dur": 13.835,
    "args": {
      "External id": 2465053,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654611.241, "dur": 12.857,
    "args": {
      "External id": 2465054,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654625.613, "dur": 12.844,
    "args": {
      "External id": 2465055,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654640.159, "dur": 22.102,
    "args": {
      "External id": 2465056,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654645.420, "dur": 1.315,
    "args": {
      "External id": 2465057,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654648.634, "dur": 0.776,
    "args": {
      "External id": 2465058,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654663.679, "dur": 11.756,
    "args": {
      "External id": 2465059,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654676.576, "dur": 10.694,
    "args": {
      "External id": 2465060,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654696.098, "dur": 1.988,
    "args": {
      "External id": 2465061,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654706.981, "dur": 3.603,
    "args": {
      "External id": 2465062,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654709.168, "dur": 0.488,
    "args": {
      "External id": 2465063,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654778.740, "dur": 53.835,
    "args": {
      "External id": 2465064,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654838.705, "dur": 3.980,
    "args": {
      "External id": 2465065,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654840.968, "dur": 0.785,
    "args": {
      "External id": 2465066,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654844.384, "dur": 23.403,
    "args": {
      "External id": 2465067,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654872.661, "dur": 4.892,
    "args": {
      "External id": 2465068,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654873.791, "dur": 3.094,
    "args": {
      "External id": 2465069,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654875.449, "dur": 1.231,
    "args": {
      "External id": 2465070,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654881.742, "dur": 43.054,
    "args": {
      "External id": 2465071,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654882.833, "dur": 41.302,
    "args": {
      "External id": 2465072,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654928.922, "dur": 13.595,
    "args": {
      "External id": 2465073,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654948.215, "dur": 3.549,
    "args": {
      "External id": 2465074,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654950.186, "dur": 0.869,
    "args": {
      "External id": 2465075,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654955.466, "dur": 86.191,
    "args": {
      "External id": 2465076,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654956.449, "dur": 5.387,
    "args": {
      "External id": 2465077,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654958.923, "dur": 2.333,
    "args": {
      "External id": 2465078,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654960.303, "dur": 0.819,
    "args": {
      "External id": 2465079,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654962.531, "dur": 78.567,
    "args": {
      "External id": 2465080,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656654963.370, "dur": 76.871,
    "args": {
      "External id": 2465081,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655048.347, "dur": 4.562,
    "args": {
      "External id": 2465082,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655050.950, "dur": 0.715,
    "args": {
      "External id": 2465083,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655059.072, "dur": 1.739,
    "args": {
      "External id": 2465084,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655070.066, "dur": 6.896,
    "args": {
      "External id": 2465085,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655072.178, "dur": 4.492,
    "args": {
      "External id": 2465086,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655159.631, "dur": 192.605,
    "args": {
      "External id": 2465087,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655163.258, "dur": 2.434,
    "args": {
      "External id": 2465088,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655182.712, "dur": 168.978,
    "args": {
      "External id": 2465089,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655184.498, "dur": 0.500,
    "args": {
      "External id": 2465090,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655186.020, "dur": 21.033,
    "args": {
      "External id": 2465091,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655208.762, "dur": 4.684,
    "args": {
      "External id": 2465092,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655212.289, "dur": 0.915,
    "args": {
      "External id": 2465093,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655216.215, "dur": 22.951,
    "args": {
      "External id": 2465094,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655217.123, "dur": 1.297,
    "args": {
      "External id": 2465095,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655219.357, "dur": 19.531,
    "args": {
      "External id": 2465096,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655222.134, "dur": 2.490,
    "args": {
      "External id": 2465097,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655240.637, "dur": 19.906,
    "args": {
      "External id": 2465098,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655261.788, "dur": 14.208,
    "args": {
      "External id": 2465099,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655278.655, "dur": 13.633,
    "args": {
      "External id": 2465100,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655293.536, "dur": 11.181,
    "args": {
      "External id": 2465101,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655306.167, "dur": 19.979,
    "args": {
      "External id": 2465102,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655309.579, "dur": 1.545,
    "args": {
      "External id": 2465103,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655312.712, "dur": 0.965,
    "args": {
      "External id": 2465104,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655327.358, "dur": 11.401,
    "args": {
      "External id": 2465105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655339.825, "dur": 10.894,
    "args": {
      "External id": 2465106,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655358.560, "dur": 1.480,
    "args": {
      "External id": 2465107,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655367.945, "dur": 3.352,
    "args": {
      "External id": 2465108,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655370.109, "dur": 0.453,
    "args": {
      "External id": 2465109,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655433.727, "dur": 53.857,
    "args": {
      "External id": 2465110,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655492.147, "dur": 4.556,
    "args": {
      "External id": 2465111,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655494.741, "dur": 0.931,
    "args": {
      "External id": 2465112,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655498.058, "dur": 24.602,
    "args": {
      "External id": 2465113,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655541.747, "dur": 8.546,
    "args": {
      "External id": 2465114,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655543.732, "dur": 5.601,
    "args": {
      "External id": 2465115,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655547.329, "dur": 1.643,
    "args": {
      "External id": 2465116,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655553.543, "dur": 79.855,
    "args": {
      "External id": 2465117,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655557.818, "dur": 74.945,
    "args": {
      "External id": 2465118,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655637.899, "dur": 34.707,
    "args": {
      "External id": 2465119,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655679.060, "dur": 3.867,
    "args": {
      "External id": 2465120,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655681.320, "dur": 0.747,
    "args": {
      "External id": 2465121,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655686.771, "dur": 62.216,
    "args": {
      "External id": 2465122,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655689.254, "dur": 2.954,
    "args": {
      "External id": 2465123,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655689.832, "dur": 1.871,
    "args": {
      "External id": 2465124,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655691.117, "dur": 0.440,
    "args": {
      "External id": 2465125,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655692.931, "dur": 55.547,
    "args": {
      "External id": 2465126,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655693.568, "dur": 54.229,
    "args": {
      "External id": 2465127,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655752.784, "dur": 3.610,
    "args": {
      "External id": 2465128,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655754.657, "dur": 0.593,
    "args": {
      "External id": 2465129,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655763.581, "dur": 1.516,
    "args": {
      "External id": 2465130,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655771.963, "dur": 6.973,
    "args": {
      "External id": 2465131,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655773.389, "dur": 5.207,
    "args": {
      "External id": 2465132,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655859.925, "dur": 203.061,
    "args": {
      "External id": 2465133,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655861.735, "dur": 2.105,
    "args": {
      "External id": 2465134,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655866.976, "dur": 195.300,
    "args": {
      "External id": 2465135,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655868.529, "dur": 0.346,
    "args": {
      "External id": 2465136,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655869.811, "dur": 20.757,
    "args": {
      "External id": 2465137,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655891.964, "dur": 2.913,
    "args": {
      "External id": 2465138,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655893.643, "dur": 0.949,
    "args": {
      "External id": 2465139,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655895.730, "dur": 21.007,
    "args": {
      "External id": 2465140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655897.022, "dur": 1.368,
    "args": {
      "External id": 2465141,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655899.373, "dur": 17.078,
    "args": {
      "External id": 2465142,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655903.637, "dur": 2.157,
    "args": {
      "External id": 2465143,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655919.693, "dur": 19.151,
    "args": {
      "External id": 2465144,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655940.020, "dur": 12.560,
    "args": {
      "External id": 2465145,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655955.144, "dur": 11.934,
    "args": {
      "External id": 2465146,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656655968.386, "dur": 11.549,
    "args": {
      "External id": 2465147,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656013.786, "dur": 21.649,
    "args": {
      "External id": 2465148,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656015.686, "dur": 2.304,
    "args": {
      "External id": 2465149,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656019.812, "dur": 1.058,
    "args": {
      "External id": 2465150,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656036.986, "dur": 11.267,
    "args": {
      "External id": 2465151,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656051.428, "dur": 9.783,
    "args": {
      "External id": 2465152,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656069.715, "dur": 2.051,
    "args": {
      "External id": 2465153,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656079.927, "dur": 3.583,
    "args": {
      "External id": 2465154,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656082.421, "dur": 0.360,
    "args": {
      "External id": 2465155,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656150.338, "dur": 52.724,
    "args": {
      "External id": 2465156,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656207.560, "dur": 3.978,
    "args": {
      "External id": 2465157,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656209.621, "dur": 0.981,
    "args": {
      "External id": 2465158,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656213.109, "dur": 22.984,
    "args": {
      "External id": 2465159,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656242.066, "dur": 5.120,
    "args": {
      "External id": 2465160,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656243.397, "dur": 3.052,
    "args": {
      "External id": 2465161,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656245.020, "dur": 1.271,
    "args": {
      "External id": 2465162,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656249.695, "dur": 38.302,
    "args": {
      "External id": 2465163,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656250.625, "dur": 36.861,
    "args": {
      "External id": 2465164,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656291.754, "dur": 13.329,
    "args": {
      "External id": 2465165,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656310.218, "dur": 5.719,
    "args": {
      "External id": 2465166,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656314.136, "dur": 1.067,
    "args": {
      "External id": 2465167,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656319.619, "dur": 46.187,
    "args": {
      "External id": 2465168,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656320.284, "dur": 3.240,
    "args": {
      "External id": 2465169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656321.085, "dur": 1.919,
    "args": {
      "External id": 2465170,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656322.255, "dur": 0.618,
    "args": {
      "External id": 2465171,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656324.157, "dur": 41.213,
    "args": {
      "External id": 2465172,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656326.409, "dur": 38.501,
    "args": {
      "External id": 2465173,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656369.873, "dur": 4.852,
    "args": {
      "External id": 2465174,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656371.578, "dur": 2.016,
    "args": {
      "External id": 2465175,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656379.684, "dur": 1.500,
    "args": {
      "External id": 2465176,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656388.001, "dur": 5.911,
    "args": {
      "External id": 2465177,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656389.980, "dur": 3.656,
    "args": {
      "External id": 2465178,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656469.633, "dur": 199.085,
    "args": {
      "External id": 2465179,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656472.068, "dur": 1.813,
    "args": {
      "External id": 2465180,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656484.364, "dur": 183.945,
    "args": {
      "External id": 2465181,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656485.570, "dur": 0.310,
    "args": {
      "External id": 2465182,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656490.165, "dur": 18.597,
    "args": {
      "External id": 2465183,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656510.040, "dur": 5.041,
    "args": {
      "External id": 2465184,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656512.040, "dur": 2.513,
    "args": {
      "External id": 2465185,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656516.020, "dur": 38.412,
    "args": {
      "External id": 2465186,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656518.759, "dur": 1.651,
    "args": {
      "External id": 2465187,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656521.384, "dur": 32.483,
    "args": {
      "External id": 2465188,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656524.057, "dur": 2.309,
    "args": {
      "External id": 2465189,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656556.495, "dur": 21.450,
    "args": {
      "External id": 2465190,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656579.290, "dur": 12.444,
    "args": {
      "External id": 2465191,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656594.296, "dur": 12.308,
    "args": {
      "External id": 2465192,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656607.998, "dur": 11.417,
    "args": {
      "External id": 2465193,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656621.108, "dur": 21.724,
    "args": {
      "External id": 2465194,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656623.301, "dur": 1.369,
    "args": {
      "External id": 2465195,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656628.369, "dur": 0.997,
    "args": {
      "External id": 2465196,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656644.222, "dur": 11.080,
    "args": {
      "External id": 2465197,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656656.336, "dur": 10.741,
    "args": {
      "External id": 2465198,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656675.885, "dur": 2.012,
    "args": {
      "External id": 2465199,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656686.059, "dur": 3.472,
    "args": {
      "External id": 2465200,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656688.191, "dur": 0.553,
    "args": {
      "External id": 2465201,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656755.132, "dur": 54.624,
    "args": {
      "External id": 2465202,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656814.522, "dur": 7.572,
    "args": {
      "External id": 2465203,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656818.558, "dur": 2.269,
    "args": {
      "External id": 2465204,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656823.707, "dur": 23.371,
    "args": {
      "External id": 2465205,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656851.937, "dur": 4.928,
    "args": {
      "External id": 2465206,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656853.151, "dur": 3.103,
    "args": {
      "External id": 2465207,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656855.211, "dur": 0.852,
    "args": {
      "External id": 2465208,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656859.275, "dur": 40.956,
    "args": {
      "External id": 2465209,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656861.905, "dur": 37.749,
    "args": {
      "External id": 2465210,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656904.124, "dur": 13.116,
    "args": {
      "External id": 2465211,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656922.629, "dur": 3.443,
    "args": {
      "External id": 2465212,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656924.508, "dur": 0.775,
    "args": {
      "External id": 2465213,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656929.719, "dur": 93.681,
    "args": {
      "External id": 2465214,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656930.367, "dur": 6.059,
    "args": {
      "External id": 2465215,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656931.445, "dur": 4.408,
    "args": {
      "External id": 2465216,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656934.715, "dur": 0.904,
    "args": {
      "External id": 2465217,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656945.090, "dur": 77.550,
    "args": {
      "External id": 2465218,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656656945.828, "dur": 75.892,
    "args": {
      "External id": 2465219,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657029.392, "dur": 4.967,
    "args": {
      "External id": 2465220,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657031.950, "dur": 1.147,
    "args": {
      "External id": 2465221,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657040.940, "dur": 1.680,
    "args": {
      "External id": 2465222,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657051.684, "dur": 8.487,
    "args": {
      "External id": 2465223,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657053.453, "dur": 6.420,
    "args": {
      "External id": 2465224,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657148.001, "dur": 181.002,
    "args": {
      "External id": 2465225,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657150.005, "dur": 2.224,
    "args": {
      "External id": 2465226,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657153.722, "dur": 174.913,
    "args": {
      "External id": 2465227,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657155.303, "dur": 0.438,
    "args": {
      "External id": 2465228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657158.832, "dur": 22.408,
    "args": {
      "External id": 2465229,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657182.832, "dur": 3.020,
    "args": {
      "External id": 2465230,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657184.693, "dur": 0.937,
    "args": {
      "External id": 2465231,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657186.847, "dur": 28.763,
    "args": {
      "External id": 2465232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657187.940, "dur": 1.819,
    "args": {
      "External id": 2465233,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657190.793, "dur": 24.442,
    "args": {
      "External id": 2465234,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657195.212, "dur": 2.441,
    "args": {
      "External id": 2465235,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657216.926, "dur": 22.937,
    "args": {
      "External id": 2465236,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657241.281, "dur": 12.884,
    "args": {
      "External id": 2465237,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657256.521, "dur": 12.483,
    "args": {
      "External id": 2465238,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657270.514, "dur": 11.752,
    "args": {
      "External id": 2465239,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657285.227, "dur": 18.165,
    "args": {
      "External id": 2465240,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657286.677, "dur": 1.278,
    "args": {
      "External id": 2465241,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657289.826, "dur": 1.066,
    "args": {
      "External id": 2465242,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657304.469, "dur": 11.249,
    "args": {
      "External id": 2465243,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657316.715, "dur": 11.077,
    "args": {
      "External id": 2465244,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657336.368, "dur": 1.478,
    "args": {
      "External id": 2465245,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657345.939, "dur": 3.305,
    "args": {
      "External id": 2465246,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657348.123, "dur": 0.343,
    "args": {
      "External id": 2465247,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657410.342, "dur": 54.184,
    "args": {
      "External id": 2465248,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657468.864, "dur": 4.770,
    "args": {
      "External id": 2465249,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657471.569, "dur": 1.030,
    "args": {
      "External id": 2465250,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657475.005, "dur": 21.527,
    "args": {
      "External id": 2465251,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657500.944, "dur": 6.773,
    "args": {
      "External id": 2465252,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657502.398, "dur": 4.633,
    "args": {
      "External id": 2465253,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657505.860, "dur": 0.997,
    "args": {
      "External id": 2465254,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657510.211, "dur": 57.447,
    "args": {
      "External id": 2465255,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657511.223, "dur": 55.336,
    "args": {
      "External id": 2465256,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657572.997, "dur": 15.110,
    "args": {
      "External id": 2465257,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657594.299, "dur": 4.684,
    "args": {
      "External id": 2465258,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657596.853, "dur": 1.049,
    "args": {
      "External id": 2465259,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657604.327, "dur": 51.110,
    "args": {
      "External id": 2465260,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657605.079, "dur": 5.860,
    "args": {
      "External id": 2465261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657606.248, "dur": 4.103,
    "args": {
      "External id": 2465262,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657607.631, "dur": 2.474,
    "args": {
      "External id": 2465263,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657611.559, "dur": 43.326,
    "args": {
      "External id": 2465264,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657612.414, "dur": 41.915,
    "args": {
      "External id": 2465265,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657659.686, "dur": 3.486,
    "args": {
      "External id": 2465266,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657661.426, "dur": 0.722,
    "args": {
      "External id": 2465267,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657670.227, "dur": 1.465,
    "args": {
      "External id": 2465268,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657679.229, "dur": 5.693,
    "args": {
      "External id": 2465269,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657680.979, "dur": 3.670,
    "args": {
      "External id": 2465270,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657764.248, "dur": 192.004,
    "args": {
      "External id": 2465271,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657766.483, "dur": 1.814,
    "args": {
      "External id": 2465272,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657769.747, "dur": 186.082,
    "args": {
      "External id": 2465273,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657772.478, "dur": 0.378,
    "args": {
      "External id": 2465274,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657775.864, "dur": 18.642,
    "args": {
      "External id": 2465275,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657796.013, "dur": 2.860,
    "args": {
      "External id": 2465276,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657797.852, "dur": 0.752,
    "args": {
      "External id": 2465277,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657799.928, "dur": 48.589,
    "args": {
      "External id": 2465278,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657800.951, "dur": 2.943,
    "args": {
      "External id": 2465279,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657804.928, "dur": 43.204,
    "args": {
      "External id": 2465280,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657834.604, "dur": 2.335,
    "args": {
      "External id": 2465281,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657849.957, "dur": 18.363,
    "args": {
      "External id": 2465282,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657869.540, "dur": 11.868,
    "args": {
      "External id": 2465283,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657885.514, "dur": 13.199,
    "args": {
      "External id": 2465284,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657899.910, "dur": 11.701,
    "args": {
      "External id": 2465285,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657912.949, "dur": 17.460,
    "args": {
      "External id": 2465286,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657914.591, "dur": 1.134,
    "args": {
      "External id": 2465287,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657917.236, "dur": 0.779,
    "args": {
      "External id": 2465288,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657931.590, "dur": 11.119,
    "args": {
      "External id": 2465289,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657943.654, "dur": 11.121,
    "args": {
      "External id": 2465290,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657963.482, "dur": 1.453,
    "args": {
      "External id": 2465291,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657972.387, "dur": 2.891,
    "args": {
      "External id": 2465292,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656657974.284, "dur": 0.332,
    "args": {
      "External id": 2465293,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658077.617, "dur": 56.971,
    "args": {
      "External id": 2465294,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658139.974, "dur": 6.716,
    "args": {
      "External id": 2465295,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658142.888, "dur": 2.325,
    "args": {
      "External id": 2465296,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658148.164, "dur": 24.195,
    "args": {
      "External id": 2465297,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658177.510, "dur": 6.654,
    "args": {
      "External id": 2465298,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658180.846, "dur": 2.717,
    "args": {
      "External id": 2465299,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658182.382, "dur": 1.005,
    "args": {
      "External id": 2465300,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658186.644, "dur": 39.076,
    "args": {
      "External id": 2465301,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658187.701, "dur": 37.546,
    "args": {
      "External id": 2465302,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658229.555, "dur": 14.095,
    "args": {
      "External id": 2465303,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658248.994, "dur": 5.453,
    "args": {
      "External id": 2465304,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658252.782, "dur": 0.934,
    "args": {
      "External id": 2465305,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658258.089, "dur": 44.731,
    "args": {
      "External id": 2465306,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658258.865, "dur": 3.114,
    "args": {
      "External id": 2465307,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658259.476, "dur": 1.944,
    "args": {
      "External id": 2465308,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658260.703, "dur": 0.595,
    "args": {
      "External id": 2465309,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658262.500, "dur": 39.977,
    "args": {
      "External id": 2465310,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658263.128, "dur": 38.866,
    "args": {
      "External id": 2465311,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658308.321, "dur": 3.904,
    "args": {
      "External id": 2465312,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658310.353, "dur": 0.945,
    "args": {
      "External id": 2465313,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658316.988, "dur": 1.369,
    "args": {
      "External id": 2465314,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658325.119, "dur": 6.193,
    "args": {
      "External id": 2465315,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658326.871, "dur": 4.199,
    "args": {
      "External id": 2465316,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658408.593, "dur": 189.785,
    "args": {
      "External id": 2465317,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658410.357, "dur": 3.407,
    "args": {
      "External id": 2465318,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658415.221, "dur": 182.599,
    "args": {
      "External id": 2465319,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658416.450, "dur": 0.550,
    "args": {
      "External id": 2465320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658418.195, "dur": 18.897,
    "args": {
      "External id": 2465321,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658438.761, "dur": 4.804,
    "args": {
      "External id": 2465322,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658442.688, "dur": 0.645,
    "args": {
      "External id": 2465323,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658444.465, "dur": 21.392,
    "args": {
      "External id": 2465324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658445.910, "dur": 1.416,
    "args": {
      "External id": 2465325,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658449.982, "dur": 15.630,
    "args": {
      "External id": 2465326,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658452.035, "dur": 2.477,
    "args": {
      "External id": 2465327,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658467.060, "dur": 17.732,
    "args": {
      "External id": 2465328,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658486.005, "dur": 12.690,
    "args": {
      "External id": 2465329,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658500.933, "dur": 12.428,
    "args": {
      "External id": 2465330,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658514.500, "dur": 10.738,
    "args": {
      "External id": 2465331,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658541.597, "dur": 23.696,
    "args": {
      "External id": 2465332,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658545.994, "dur": 1.710,
    "args": {
      "External id": 2465333,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658549.656, "dur": 0.883,
    "args": {
      "External id": 2465334,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658572.840, "dur": 11.580,
    "args": {
      "External id": 2465335,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658585.472, "dur": 11.248,
    "args": {
      "External id": 2465336,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658605.172, "dur": 2.026,
    "args": {
      "External id": 2465337,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658615.561, "dur": 3.142,
    "args": {
      "External id": 2465338,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658617.465, "dur": 0.378,
    "args": {
      "External id": 2465339,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658680.140, "dur": 51.333,
    "args": {
      "External id": 2465340,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658736.332, "dur": 5.738,
    "args": {
      "External id": 2465341,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658740.103, "dur": 0.914,
    "args": {
      "External id": 2465342,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658743.484, "dur": 22.478,
    "args": {
      "External id": 2465343,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658770.466, "dur": 4.747,
    "args": {
      "External id": 2465344,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658771.658, "dur": 3.036,
    "args": {
      "External id": 2465345,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658773.349, "dur": 1.173,
    "args": {
      "External id": 2465346,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658777.317, "dur": 41.962,
    "args": {
      "External id": 2465347,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658778.465, "dur": 40.108,
    "args": {
      "External id": 2465348,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658824.829, "dur": 13.852,
    "args": {
      "External id": 2465349,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658843.574, "dur": 22.553,
    "args": {
      "External id": 2465350,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658845.851, "dur": 19.910,
    "args": {
      "External id": 2465351,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658852.083, "dur": 0.639,
    "args": {
      "External id": 2465352,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658870.774, "dur": 24.726,
    "args": {
      "External id": 2465353,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658872.757, "dur": 22.543,
    "args": {
      "External id": 2465354,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658876.972, "dur": 4.158,
    "args": {
      "External id": 2465355,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658882.260, "dur": 12.536,
    "args": {
      "External id": 2465356,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658906.172, "dur": 6.414,
    "args": {
      "External id": 2465357,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658909.437, "dur": 2.724,
    "args": {
      "External id": 2465358,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658913.721, "dur": 1.122,
    "args": {
      "External id": 2465359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658914.208, "dur": 0.554,
    "args": {
      "External id": 2465360,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658955.744, "dur": 20.001,
    "args": {
      "External id": 2465361,"Sequence number": 24740640, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656658977.409, "dur": 52.898,
    "args": {
      "External id": 2465362,"Sequence number": 24740641, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9961
    }
  },
  {
    "ph": "s", "id": 9, "pid": 1336756, "tid": 1336756, "ts": 1514656658977.409,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659039.350, "dur": 7.583,
    "args": {
      "External id": 2465363,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659043.662, "dur": 1.527,
    "args": {
      "External id": 2465364,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659049.440, "dur": 13.181,
    "args": {
      "External id": 2465365,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "3"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659057.696, "dur": 3.605,
    "args": {
      "External id": 2465366,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659064.081, "dur": 3.123,
    "args": {
      "External id": 2465367,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659065.564, "dur": 1.032,
    "args": {
      "External id": 2465368,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659071.891, "dur": 7.746,
    "args": {
      "External id": 2465369,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9968
    }
  },
  {
    "ph": "s", "id": 8, "pid": 1336756, "tid": 1336756, "ts": 1514656659071.891,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659077.214, "dur": 1.024,
    "args": {
      "External id": 2465370,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659080.800, "dur": 4.140,
    "args": {
      "External id": 2465371,"Sequence number": 24740643, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9970
    }
  },
  {
    "ph": "s", "id": 7, "pid": 1336756, "tid": 1336756, "ts": 1514656659080.800,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659083.628, "dur": 0.552,
    "args": {
      "External id": 2465372,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659086.017, "dur": 6.803,
    "args": {
      "External id": 2465373,"Sequence number": 24740644, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9972
    }
  },
  {
    "ph": "s", "id": 6, "pid": 1336756, "tid": 1336756, "ts": 1514656659086.017,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659091.071, "dur": 0.775,
    "args": {
      "External id": 2465374,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659094.072, "dur": 5.774,
    "args": {
      "External id": 2465375,"Sequence number": 24740645, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9974
    }
  },
  {
    "ph": "s", "id": 5, "pid": 1336756, "tid": 1336756, "ts": 1514656659094.072,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659096.786, "dur": 2.296,
    "args": {
      "External id": 2465376,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659103.941, "dur": 33.544,
    "args": {
      "External id": 2465377,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659105.451, "dur": 31.788,
    "args": {
      "External id": 2465378,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659108.046, "dur": 7.000,
    "args": {
      "External id": 2465379,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659110.352, "dur": 3.894,
    "args": {
      "External id": 2465380,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659116.266, "dur": 20.567,
    "args": {
      "External id": 2465381,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659163.381, "dur": 3.526,
    "args": {
      "External id": 2465382,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9981
    }
  },
  {
    "ph": "s", "id": 4, "pid": 1336756, "tid": 1336756, "ts": 1514656659163.381,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659170.947, "dur": 1.164,
    "args": {
      "External id": 2465383,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659204.323, "dur": 43830.537,
    "args": {
      "External id": 2465384,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9983
    }
  },
  {
    "ph": "s", "id": 3, "pid": 1336756, "tid": 1336756, "ts": 1514656659204.323,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659218.296, "dur": 25.571,
    "args": {
      "External id": 2465385,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659218.987, "dur": 24.592,
    "args": {
      "External id": 2465386,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659220.598, "dur": 7.052,
    "args": {
      "External id": 2465387,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659221.723, "dur": 5.483,
    "args": {
      "External id": 2465388,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659228.314, "dur": 14.680,
    "args": {
      "External id": 2465389,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659258.925, "dur": 25.957,
    "args": {
      "External id": 2465390,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659259.917, "dur": 6.064,
    "args": {
      "External id": 2465391,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659261.745, "dur": 3.937,
    "args": {
      "External id": 2465392,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659268.832, "dur": 15.838,
    "args": {
      "External id": 2465393,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659270.693, "dur": 13.428,
    "args": {
      "External id": 2465394,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659288.241, "dur": 17.869,
    "args": {
      "External id": 2465395,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659289.189, "dur": 4.174,
    "args": {
      "External id": 2465396,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659290.526, "dur": 2.559,
    "args": {
      "External id": 2465397,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659293.826, "dur": 12.096,
    "args": {
      "External id": 2465398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659294.383, "dur": 11.219,
    "args": {
      "External id": 2465399,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659311.714, "dur": 18.627,
    "args": {
      "External id": 2465400,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659313.269, "dur": 3.593,
    "args": {
      "External id": 2465401,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659319.420, "dur": 10.667,
    "args": {
      "External id": 2465402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 10001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659320.165, "dur": 9.516,
    "args": {
      "External id": 2465403,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659335.142, "dur": 20.975,
    "args": {
      "External id": 2465404,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659358.505, "dur": 45.985,
    "args": {
      "External id": 2465405,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659360.296, "dur": 43.731,
    "args": {
      "External id": 2465406,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659366.701, "dur": 0.710,
    "args": {
      "External id": 2465407,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659368.735, "dur": 21.252,
    "args": {
      "External id": 2465408,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659370.376, "dur": 19.229,
    "args": {
      "External id": 2465409,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 10008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659372.659, "dur": 2.483,
    "args": {
      "External id": 2465410,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659376.084, "dur": 13.107,
    "args": {
      "External id": 2465411,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 10010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659408.169, "dur": 38221.799,
    "args": {
      "External id": 2465412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1336756,
    "ts": 1514656659409.730, "dur": 38219.366,
    "args": {
      "External id": 2465413,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697641.387, "dur": 5.875,
    "args": {
      "External id": 2465414,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697644.529, "dur": 1.000,
    "args": {
      "External id": 2465415,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697652.000, "dur": 103.316,
    "args": {
      "External id": 2465416,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697653.506, "dur": 7.784,
    "args": {
      "External id": 2465417,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697655.361, "dur": 5.033,
    "args": {
      "External id": 2465418,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697659.126, "dur": 0.993,
    "args": {
      "External id": 2465419,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697662.578, "dur": 92.174,
    "args": {
      "External id": 2465420,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697664.326, "dur": 89.557,
    "args": {
      "External id": 2465421,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697758.918, "dur": 4.281,
    "args": {
      "External id": 2465422,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697760.804, "dur": 1.119,
    "args": {
      "External id": 2465423,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697769.502, "dur": 1.989,
    "args": {
      "External id": 2465424,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697780.122, "dur": 7.486,
    "args": {
      "External id": 2465425,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697783.509, "dur": 3.832,
    "args": {
      "External id": 2465426,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697913.551, "dur": 226.272,
    "args": {
      "External id": 2465427,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697917.356, "dur": 2.155,
    "args": {
      "External id": 2465428,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697921.416, "dur": 217.871,
    "args": {
      "External id": 2465429,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697922.782, "dur": 0.416,
    "args": {
      "External id": 2465430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697924.816, "dur": 23.544,
    "args": {
      "External id": 2465431,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697950.237, "dur": 4.785,
    "args": {
      "External id": 2465432,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697954.030, "dur": 0.703,
    "args": {
      "External id": 2465433,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697957.891, "dur": 52.113,
    "args": {
      "External id": 2465434,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697959.206, "dur": 1.611,
    "args": {
      "External id": 2465435,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697962.060, "dur": 47.208,
    "args": {
      "External id": 2465436,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656697965.666, "dur": 3.765,
    "args": {
      "External id": 2465437,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698012.168, "dur": 24.752,
    "args": {
      "External id": 2465438,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698038.752, "dur": 17.223,
    "args": {
      "External id": 2465439,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698059.504, "dur": 13.730,
    "args": {
      "External id": 2465440,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698075.090, "dur": 13.039,
    "args": {
      "External id": 2465441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698090.176, "dur": 22.611,
    "args": {
      "External id": 2465442,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698094.087, "dur": 2.023,
    "args": {
      "External id": 2465443,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698098.447, "dur": 0.844,
    "args": {
      "External id": 2465444,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698114.366, "dur": 11.731,
    "args": {
      "External id": 2465445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698127.562, "dur": 10.455,
    "args": {
      "External id": 2465446,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698147.512, "dur": 2.495,
    "args": {
      "External id": 2465447,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698157.003, "dur": 4.351,
    "args": {
      "External id": 2465448,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698159.378, "dur": 0.925,
    "args": {
      "External id": 2465449,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698241.405, "dur": 67.132,
    "args": {
      "External id": 2465450,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698314.129, "dur": 5.301,
    "args": {
      "External id": 2465451,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698316.353, "dur": 0.815,
    "args": {
      "External id": 2465452,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698320.957, "dur": 25.036,
    "args": {
      "External id": 2465453,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698351.624, "dur": 8.109,
    "args": {
      "External id": 2465454,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698353.530, "dur": 5.450,
    "args": {
      "External id": 2465455,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698357.337, "dur": 1.403,
    "args": {
      "External id": 2465456,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698362.795, "dur": 43.326,
    "args": {
      "External id": 2465457,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698363.842, "dur": 41.642,
    "args": {
      "External id": 2465458,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698410.229, "dur": 14.045,
    "args": {
      "External id": 2465459,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698429.927, "dur": 3.468,
    "args": {
      "External id": 2465460,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698431.915, "dur": 0.557,
    "args": {
      "External id": 2465461,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698437.609, "dur": 48.633,
    "args": {
      "External id": 2465462,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698439.976, "dur": 4.003,
    "args": {
      "External id": 2465463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698441.011, "dur": 2.392,
    "args": {
      "External id": 2465464,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698442.377, "dur": 0.661,
    "args": {
      "External id": 2465465,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698444.557, "dur": 41.339,
    "args": {
      "External id": 2465466,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698445.138, "dur": 40.190,
    "args": {
      "External id": 2465467,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698489.887, "dur": 3.187,
    "args": {
      "External id": 2465468,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698491.508, "dur": 0.533,
    "args": {
      "External id": 2465469,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698500.145, "dur": 1.573,
    "args": {
      "External id": 2465470,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698509.714, "dur": 7.052,
    "args": {
      "External id": 2465471,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698511.462, "dur": 5.006,
    "args": {
      "External id": 2465472,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698627.985, "dur": 180.985,
    "args": {
      "External id": 2465473,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698630.431, "dur": 3.007,
    "args": {
      "External id": 2465474,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698636.938, "dur": 171.524,
    "args": {
      "External id": 2465475,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698638.580, "dur": 0.320,
    "args": {
      "External id": 2465476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698640.277, "dur": 23.938,
    "args": {
      "External id": 2465477,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698665.637, "dur": 3.596,
    "args": {
      "External id": 2465478,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698667.814, "dur": 1.115,
    "args": {
      "External id": 2465479,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698670.272, "dur": 22.730,
    "args": {
      "External id": 2465480,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698671.171, "dur": 1.737,
    "args": {
      "External id": 2465481,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698674.220, "dur": 18.404,
    "args": {
      "External id": 2465482,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698678.766, "dur": 2.587,
    "args": {
      "External id": 2465483,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698696.304, "dur": 21.241,
    "args": {
      "External id": 2465484,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698719.085, "dur": 12.634,
    "args": {
      "External id": 2465485,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698734.568, "dur": 13.494,
    "args": {
      "External id": 2465486,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698749.651, "dur": 12.276,
    "args": {
      "External id": 2465487,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698763.630, "dur": 18.503,
    "args": {
      "External id": 2465488,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698765.576, "dur": 1.461,
    "args": {
      "External id": 2465489,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698768.608, "dur": 0.725,
    "args": {
      "External id": 2465490,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698783.477, "dur": 11.257,
    "args": {
      "External id": 2465491,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698797.434, "dur": 10.175,
    "args": {
      "External id": 2465492,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698815.835, "dur": 1.934,
    "args": {
      "External id": 2465493,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698825.944, "dur": 3.152,
    "args": {
      "External id": 2465494,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698828.030, "dur": 0.362,
    "args": {
      "External id": 2465495,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698893.081, "dur": 53.762,
    "args": {
      "External id": 2465496,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698951.997, "dur": 4.331,
    "args": {
      "External id": 2465497,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698954.490, "dur": 0.903,
    "args": {
      "External id": 2465498,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656698957.689, "dur": 22.837,
    "args": {
      "External id": 2465499,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699021.781, "dur": 5.770,
    "args": {
      "External id": 2465500,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699023.355, "dur": 3.399,
    "args": {
      "External id": 2465501,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699025.140, "dur": 1.307,
    "args": {
      "External id": 2465502,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699030.591, "dur": 47.133,
    "args": {
      "External id": 2465503,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699031.696, "dur": 45.419,
    "args": {
      "External id": 2465504,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699082.121, "dur": 15.201,
    "args": {
      "External id": 2465505,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699103.856, "dur": 5.686,
    "args": {
      "External id": 2465506,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699107.825, "dur": 0.816,
    "args": {
      "External id": 2465507,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699113.342, "dur": 51.025,
    "args": {
      "External id": 2465508,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699114.255, "dur": 5.807,
    "args": {
      "External id": 2465509,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699115.112, "dur": 4.340,
    "args": {
      "External id": 2465510,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699116.371, "dur": 2.668,
    "args": {
      "External id": 2465511,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699120.603, "dur": 43.164,
    "args": {
      "External id": 2465512,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699122.859, "dur": 40.377,
    "args": {
      "External id": 2465513,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699168.487, "dur": 3.637,
    "args": {
      "External id": 2465514,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699170.476, "dur": 0.759,
    "args": {
      "External id": 2465515,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699177.910, "dur": 1.406,
    "args": {
      "External id": 2465516,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699186.813, "dur": 5.547,
    "args": {
      "External id": 2465517,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699188.567, "dur": 3.527,
    "args": {
      "External id": 2465518,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699282.046, "dur": 171.005,
    "args": {
      "External id": 2465519,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699284.180, "dur": 1.928,
    "args": {
      "External id": 2465520,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699287.422, "dur": 165.168,
    "args": {
      "External id": 2465521,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699288.680, "dur": 0.258,
    "args": {
      "External id": 2465522,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699291.687, "dur": 19.373,
    "args": {
      "External id": 2465523,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699312.744, "dur": 2.563,
    "args": {
      "External id": 2465524,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699314.403, "dur": 0.646,
    "args": {
      "External id": 2465525,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699316.450, "dur": 21.842,
    "args": {
      "External id": 2465526,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699317.562, "dur": 2.943,
    "args": {
      "External id": 2465527,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699321.858, "dur": 16.145,
    "args": {
      "External id": 2465528,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699324.516, "dur": 2.837,
    "args": {
      "External id": 2465529,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699341.326, "dur": 22.380,
    "args": {
      "External id": 2465530,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699365.099, "dur": 12.825,
    "args": {
      "External id": 2465531,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699380.427, "dur": 12.376,
    "args": {
      "External id": 2465532,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699394.296, "dur": 11.699,
    "args": {
      "External id": 2465533,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699407.450, "dur": 17.958,
    "args": {
      "External id": 2465534,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699409.377, "dur": 1.352,
    "args": {
      "External id": 2465535,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699412.415, "dur": 0.760,
    "args": {
      "External id": 2465536,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699427.925, "dur": 11.958,
    "args": {
      "External id": 2465537,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699440.797, "dur": 10.842,
    "args": {
      "External id": 2465538,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699459.357, "dur": 1.893,
    "args": {
      "External id": 2465539,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699469.381, "dur": 2.913,
    "args": {
      "External id": 2465540,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699471.243, "dur": 0.333,
    "args": {
      "External id": 2465541,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699552.445, "dur": 55.314,
    "args": {
      "External id": 2465542,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699613.622, "dur": 5.350,
    "args": {
      "External id": 2465543,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699616.665, "dur": 0.944,
    "args": {
      "External id": 2465544,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699621.898, "dur": 30.591,
    "args": {
      "External id": 2465545,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699657.241, "dur": 7.108,
    "args": {
      "External id": 2465546,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699658.684, "dur": 4.986,
    "args": {
      "External id": 2465547,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699660.547, "dur": 2.938,
    "args": {
      "External id": 2465548,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699666.628, "dur": 72.553,
    "args": {
      "External id": 2465549,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699667.496, "dur": 70.925,
    "args": {
      "External id": 2465550,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699742.733, "dur": 35.507,
    "args": {
      "External id": 2465551,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699785.377, "dur": 3.467,
    "args": {
      "External id": 2465552,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699787.141, "dur": 0.868,
    "args": {
      "External id": 2465553,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699792.526, "dur": 63.599,
    "args": {
      "External id": 2465554,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699793.487, "dur": 3.743,
    "args": {
      "External id": 2465555,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699794.339, "dur": 2.384,
    "args": {
      "External id": 2465556,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699795.809, "dur": 0.770,
    "args": {
      "External id": 2465557,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699799.512, "dur": 56.144,
    "args": {
      "External id": 2465558,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699799.976, "dur": 55.010,
    "args": {
      "External id": 2465559,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699860.193, "dur": 3.895,
    "args": {
      "External id": 2465560,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699862.168, "dur": 0.974,
    "args": {
      "External id": 2465561,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699869.387, "dur": 1.398,
    "args": {
      "External id": 2465562,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699877.899, "dur": 8.200,
    "args": {
      "External id": 2465563,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699881.584, "dur": 4.150,
    "args": {
      "External id": 2465564,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699967.169, "dur": 223.496,
    "args": {
      "External id": 2465565,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699968.785, "dur": 2.055,
    "args": {
      "External id": 2465566,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699973.954, "dur": 216.090,
    "args": {
      "External id": 2465567,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656699975.082, "dur": 0.411,
    "args": {
      "External id": 2465568,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700023.691, "dur": 23.908,
    "args": {
      "External id": 2465569,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700050.014, "dur": 4.934,
    "args": {
      "External id": 2465570,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700052.044, "dur": 2.595,
    "args": {
      "External id": 2465571,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700057.664, "dur": 19.925,
    "args": {
      "External id": 2465572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700059.005, "dur": 1.439,
    "args": {
      "External id": 2465573,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700061.705, "dur": 15.600,
    "args": {
      "External id": 2465574,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700064.261, "dur": 2.135,
    "args": {
      "External id": 2465575,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700078.899, "dur": 18.948,
    "args": {
      "External id": 2465576,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700099.313, "dur": 12.252,
    "args": {
      "External id": 2465577,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700114.499, "dur": 13.624,
    "args": {
      "External id": 2465578,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700129.527, "dur": 12.193,
    "args": {
      "External id": 2465579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700143.293, "dur": 20.987,
    "args": {
      "External id": 2465580,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700146.852, "dur": 1.209,
    "args": {
      "External id": 2465581,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700149.693, "dur": 2.093,
    "args": {
      "External id": 2465582,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700165.485, "dur": 11.263,
    "args": {
      "External id": 2465583,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700178.188, "dur": 10.924,
    "args": {
      "External id": 2465584,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700197.872, "dur": 2.447,
    "args": {
      "External id": 2465585,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700208.452, "dur": 3.664,
    "args": {
      "External id": 2465586,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700210.792, "dur": 0.536,
    "args": {
      "External id": 2465587,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700278.146, "dur": 54.781,
    "args": {
      "External id": 2465588,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700339.709, "dur": 4.105,
    "args": {
      "External id": 2465589,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700342.017, "dur": 0.840,
    "args": {
      "External id": 2465590,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700345.020, "dur": 22.713,
    "args": {
      "External id": 2465591,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700372.093, "dur": 5.538,
    "args": {
      "External id": 2465592,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700373.653, "dur": 3.331,
    "args": {
      "External id": 2465593,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700375.281, "dur": 1.527,
    "args": {
      "External id": 2465594,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700381.655, "dur": 39.118,
    "args": {
      "External id": 2465595,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700382.786, "dur": 37.306,
    "args": {
      "External id": 2465596,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700424.445, "dur": 13.747,
    "args": {
      "External id": 2465597,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700443.471, "dur": 3.731,
    "args": {
      "External id": 2465598,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700445.494, "dur": 0.804,
    "args": {
      "External id": 2465599,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700450.704, "dur": 47.435,
    "args": {
      "External id": 2465600,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700451.678, "dur": 5.595,
    "args": {
      "External id": 2465601,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700454.573, "dur": 2.202,
    "args": {
      "External id": 2465602,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700455.856, "dur": 0.770,
    "args": {
      "External id": 2465603,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700457.871, "dur": 39.850,
    "args": {
      "External id": 2465604,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700458.499, "dur": 38.771,
    "args": {
      "External id": 2465605,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700502.260, "dur": 3.800,
    "args": {
      "External id": 2465606,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700504.142, "dur": 0.887,
    "args": {
      "External id": 2465607,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700510.905, "dur": 1.430,
    "args": {
      "External id": 2465608,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700520.314, "dur": 5.770,
    "args": {
      "External id": 2465609,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700522.029, "dur": 3.789,
    "args": {
      "External id": 2465610,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700627.482, "dur": 180.699,
    "args": {
      "External id": 2465611,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700631.615, "dur": 2.698,
    "args": {
      "External id": 2465612,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700635.928, "dur": 171.686,
    "args": {
      "External id": 2465613,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700637.201, "dur": 0.301,
    "args": {
      "External id": 2465614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700638.407, "dur": 21.395,
    "args": {
      "External id": 2465615,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700662.878, "dur": 5.102,
    "args": {
      "External id": 2465616,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700666.560, "dur": 1.133,
    "args": {
      "External id": 2465617,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700673.211, "dur": 22.619,
    "args": {
      "External id": 2465618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700676.709, "dur": 1.378,
    "args": {
      "External id": 2465619,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700679.431, "dur": 16.125,
    "args": {
      "External id": 2465620,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700682.341, "dur": 2.110,
    "args": {
      "External id": 2465621,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700697.202, "dur": 19.525,
    "args": {
      "External id": 2465622,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700718.051, "dur": 14.700,
    "args": {
      "External id": 2465623,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700735.415, "dur": 12.851,
    "args": {
      "External id": 2465624,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700749.440, "dur": 11.738,
    "args": {
      "External id": 2465625,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700764.142, "dur": 18.123,
    "args": {
      "External id": 2465626,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700766.011, "dur": 1.144,
    "args": {
      "External id": 2465627,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700769.055, "dur": 0.623,
    "args": {
      "External id": 2465628,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700783.419, "dur": 11.527,
    "args": {
      "External id": 2465629,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700796.067, "dur": 10.640,
    "args": {
      "External id": 2465630,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700814.934, "dur": 1.794,
    "args": {
      "External id": 2465631,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700825.139, "dur": 3.473,
    "args": {
      "External id": 2465632,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700827.274, "dur": 0.617,
    "args": {
      "External id": 2465633,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700893.913, "dur": 51.819,
    "args": {
      "External id": 2465634,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700950.704, "dur": 4.203,
    "args": {
      "External id": 2465635,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700953.059, "dur": 0.781,
    "args": {
      "External id": 2465636,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656700956.359, "dur": 22.771,
    "args": {
      "External id": 2465637,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701025.904, "dur": 7.769,
    "args": {
      "External id": 2465638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701027.428, "dur": 5.375,
    "args": {
      "External id": 2465639,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701030.940, "dur": 1.516,
    "args": {
      "External id": 2465640,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701036.522, "dur": 48.215,
    "args": {
      "External id": 2465641,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701037.765, "dur": 46.215,
    "args": {
      "External id": 2465642,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701088.980, "dur": 15.211,
    "args": {
      "External id": 2465643,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701110.206, "dur": 3.712,
    "args": {
      "External id": 2465644,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701112.329, "dur": 0.726,
    "args": {
      "External id": 2465645,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701117.798, "dur": 48.621,
    "args": {
      "External id": 2465646,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701120.299, "dur": 3.718,
    "args": {
      "External id": 2465647,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701121.081, "dur": 2.381,
    "args": {
      "External id": 2465648,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701122.521, "dur": 0.817,
    "args": {
      "External id": 2465649,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701124.535, "dur": 41.403,
    "args": {
      "External id": 2465650,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701125.240, "dur": 40.132,
    "args": {
      "External id": 2465651,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701170.419, "dur": 3.424,
    "args": {
      "External id": 2465652,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701172.425, "dur": 0.437,
    "args": {
      "External id": 2465653,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701181.415, "dur": 1.682,
    "args": {
      "External id": 2465654,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701190.442, "dur": 7.777,
    "args": {
      "External id": 2465655,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701192.209, "dur": 5.672,
    "args": {
      "External id": 2465656,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701282.008, "dur": 176.844,
    "args": {
      "External id": 2465657,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701284.182, "dur": 2.033,
    "args": {
      "External id": 2465658,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701289.583, "dur": 168.774,
    "args": {
      "External id": 2465659,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701291.383, "dur": 0.251,
    "args": {
      "External id": 2465660,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701292.559, "dur": 20.300,
    "args": {
      "External id": 2465661,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701314.293, "dur": 3.316,
    "args": {
      "External id": 2465662,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701316.227, "dur": 1.098,
    "args": {
      "External id": 2465663,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701318.575, "dur": 23.225,
    "args": {
      "External id": 2465664,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701319.754, "dur": 2.052,
    "args": {
      "External id": 2465665,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701322.842, "dur": 18.641,
    "args": {
      "External id": 2465666,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701327.158, "dur": 2.859,
    "args": {
      "External id": 2465667,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701343.102, "dur": 23.416,
    "args": {
      "External id": 2465668,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701367.852, "dur": 13.610,
    "args": {
      "External id": 2465669,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701385.892, "dur": 13.614,
    "args": {
      "External id": 2465670,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701400.838, "dur": 11.501,
    "args": {
      "External id": 2465671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701413.957, "dur": 18.432,
    "args": {
      "External id": 2465672,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701415.716, "dur": 1.724,
    "args": {
      "External id": 2465673,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701419.070, "dur": 0.902,
    "args": {
      "External id": 2465674,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701433.676, "dur": 11.377,
    "args": {
      "External id": 2465675,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701448.025, "dur": 9.421,
    "args": {
      "External id": 2465676,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701464.759, "dur": 1.329,
    "args": {
      "External id": 2465677,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701473.537, "dur": 2.890,
    "args": {
      "External id": 2465678,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701475.306, "dur": 0.441,
    "args": {
      "External id": 2465679,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701552.363, "dur": 53.114,
    "args": {
      "External id": 2465680,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701611.021, "dur": 5.653,
    "args": {
      "External id": 2465681,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701614.101, "dur": 1.220,
    "args": {
      "External id": 2465682,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701618.458, "dur": 24.214,
    "args": {
      "External id": 2465683,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701649.092, "dur": 4.707,
    "args": {
      "External id": 2465684,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701650.619, "dur": 2.594,
    "args": {
      "External id": 2465685,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701652.216, "dur": 0.814,
    "args": {
      "External id": 2465686,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701656.043, "dur": 42.000,
    "args": {
      "External id": 2465687,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701657.058, "dur": 40.324,
    "args": {
      "External id": 2465688,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701701.564, "dur": 13.990,
    "args": {
      "External id": 2465689,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701720.782, "dur": 4.829,
    "args": {
      "External id": 2465690,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701724.295, "dur": 0.539,
    "args": {
      "External id": 2465691,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701729.504, "dur": 48.406,
    "args": {
      "External id": 2465692,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701730.376, "dur": 4.698,
    "args": {
      "External id": 2465693,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701730.906, "dur": 3.595,
    "args": {
      "External id": 2465694,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701732.183, "dur": 2.136,
    "args": {
      "External id": 2465695,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701735.531, "dur": 41.906,
    "args": {
      "External id": 2465696,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701737.451, "dur": 39.511,
    "args": {
      "External id": 2465697,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701782.086, "dur": 3.566,
    "args": {
      "External id": 2465698,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701783.979, "dur": 0.594,
    "args": {
      "External id": 2465699,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701790.776, "dur": 1.437,
    "args": {
      "External id": 2465700,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701798.728, "dur": 5.130,
    "args": {
      "External id": 2465701,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701800.186, "dur": 3.369,
    "args": {
      "External id": 2465702,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701881.473, "dur": 204.520,
    "args": {
      "External id": 2465703,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701883.463, "dur": 1.843,
    "args": {
      "External id": 2465704,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701886.998, "dur": 198.513,
    "args": {
      "External id": 2465705,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701887.857, "dur": 0.383,
    "args": {
      "External id": 2465706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701891.000, "dur": 18.797,
    "args": {
      "External id": 2465707,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701911.126, "dur": 2.680,
    "args": {
      "External id": 2465708,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701912.876, "dur": 0.567,
    "args": {
      "External id": 2465709,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701914.908, "dur": 22.467,
    "args": {
      "External id": 2465710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701916.188, "dur": 3.016,
    "args": {
      "External id": 2465711,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701920.432, "dur": 16.693,
    "args": {
      "External id": 2465712,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701924.425, "dur": 2.227,
    "args": {
      "External id": 2465713,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701938.514, "dur": 19.171,
    "args": {
      "External id": 2465714,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701958.870, "dur": 11.968,
    "args": {
      "External id": 2465715,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656701973.148, "dur": 48.020,
    "args": {
      "External id": 2465716,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702023.400, "dur": 12.975,
    "args": {
      "External id": 2465717,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702038.392, "dur": 19.835,
    "args": {
      "External id": 2465718,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702040.408, "dur": 1.687,
    "args": {
      "External id": 2465719,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702043.996, "dur": 0.797,
    "args": {
      "External id": 2465720,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702061.594, "dur": 10.792,
    "args": {
      "External id": 2465721,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702073.413, "dur": 11.107,
    "args": {
      "External id": 2465722,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702092.853, "dur": 2.244,
    "args": {
      "External id": 2465723,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702104.457, "dur": 3.522,
    "args": {
      "External id": 2465724,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702106.863, "dur": 0.368,
    "args": {
      "External id": 2465725,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702170.695, "dur": 50.709,
    "args": {
      "External id": 2465726,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702226.057, "dur": 4.440,
    "args": {
      "External id": 2465727,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702228.710, "dur": 0.962,
    "args": {
      "External id": 2465728,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702233.579, "dur": 21.463,
    "args": {
      "External id": 2465729,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702259.501, "dur": 5.252,
    "args": {
      "External id": 2465730,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702261.002, "dur": 3.044,
    "args": {
      "External id": 2465731,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702262.736, "dur": 1.118,
    "args": {
      "External id": 2465732,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702267.283, "dur": 39.610,
    "args": {
      "External id": 2465733,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702268.511, "dur": 37.678,
    "args": {
      "External id": 2465734,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702310.435, "dur": 12.917,
    "args": {
      "External id": 2465735,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702329.982, "dur": 3.942,
    "args": {
      "External id": 2465736,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702332.336, "dur": 0.883,
    "args": {
      "External id": 2465737,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702337.457, "dur": 49.250,
    "args": {
      "External id": 2465738,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702338.314, "dur": 5.274,
    "args": {
      "External id": 2465739,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702339.134, "dur": 3.877,
    "args": {
      "External id": 2465740,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702342.302, "dur": 0.604,
    "args": {
      "External id": 2465741,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702345.878, "dur": 40.425,
    "args": {
      "External id": 2465742,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702346.567, "dur": 39.072,
    "args": {
      "External id": 2465743,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702390.649, "dur": 3.590,
    "args": {
      "External id": 2465744,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702392.438, "dur": 0.885,
    "args": {
      "External id": 2465745,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702398.953, "dur": 1.447,
    "args": {
      "External id": 2465746,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702406.881, "dur": 6.899,
    "args": {
      "External id": 2465747,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702410.207, "dur": 3.282,
    "args": {
      "External id": 2465748,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702484.181, "dur": 183.142,
    "args": {
      "External id": 2465749,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702486.558, "dur": 1.973,
    "args": {
      "External id": 2465750,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702489.625, "dur": 177.316,
    "args": {
      "External id": 2465751,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702492.389, "dur": 0.264,
    "args": {
      "External id": 2465752,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702493.830, "dur": 17.396,
    "args": {
      "External id": 2465753,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702512.660, "dur": 3.218,
    "args": {
      "External id": 2465754,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702514.643, "dur": 0.902,
    "args": {
      "External id": 2465755,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702519.029, "dur": 38.404,
    "args": {
      "External id": 2465756,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702521.543, "dur": 1.358,
    "args": {
      "External id": 2465757,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702523.908, "dur": 33.216,
    "args": {
      "External id": 2465758,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702526.206, "dur": 17.886,
    "args": {
      "External id": 2465759,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702559.546, "dur": 18.318,
    "args": {
      "External id": 2465760,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702579.026, "dur": 13.150,
    "args": {
      "External id": 2465761,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702594.488, "dur": 12.171,
    "args": {
      "External id": 2465762,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702607.756, "dur": 11.651,
    "args": {
      "External id": 2465763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702621.044, "dur": 20.295,
    "args": {
      "External id": 2465764,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702624.598, "dur": 1.205,
    "args": {
      "External id": 2465765,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702627.724, "dur": 1.008,
    "args": {
      "External id": 2465766,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702642.606, "dur": 11.050,
    "args": {
      "External id": 2465767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702654.717, "dur": 10.897,
    "args": {
      "External id": 2465768,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702673.744, "dur": 2.216,
    "args": {
      "External id": 2465769,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702683.869, "dur": 3.518,
    "args": {
      "External id": 2465770,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702686.073, "dur": 0.602,
    "args": {
      "External id": 2465771,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702748.406, "dur": 48.991,
    "args": {
      "External id": 2465772,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702803.309, "dur": 5.665,
    "args": {
      "External id": 2465773,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702805.443, "dur": 2.504,
    "args": {
      "External id": 2465774,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702810.602, "dur": 22.092,
    "args": {
      "External id": 2465775,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702836.866, "dur": 4.448,
    "args": {
      "External id": 2465776,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702838.009, "dur": 2.762,
    "args": {
      "External id": 2465777,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702839.965, "dur": 0.662,
    "args": {
      "External id": 2465778,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702845.357, "dur": 38.037,
    "args": {
      "External id": 2465779,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702846.504, "dur": 36.396,
    "args": {
      "External id": 2465780,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702886.600, "dur": 13.025,
    "args": {
      "External id": 2465781,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702904.271, "dur": 21.005,
    "args": {
      "External id": 2465782,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702906.238, "dur": 18.661,
    "args": {
      "External id": 2465783,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702911.148, "dur": 0.868,
    "args": {
      "External id": 2465784,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702930.008, "dur": 26.126,
    "args": {
      "External id": 2465785,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702931.758, "dur": 24.176,
    "args": {
      "External id": 2465786,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 10385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702935.761, "dur": 4.017,
    "args": {
      "External id": 2465787,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702942.789, "dur": 12.640,
    "args": {
      "External id": 2465788,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702967.854, "dur": 4.955,
    "args": {
      "External id": 2465789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702969.883, "dur": 2.700,
    "args": {
      "External id": 2465790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702974.119, "dur": 0.827,
    "args": {
      "External id": 2465791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336756, "tid": 1336756,
    "ts": 1514656702974.414, "dur": 0.452,
    "args": {
      "External id": 2465792,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703092.835, "dur": 30.925,
    "args": {
      "External id": 2465793,"Sequence number": 24740648, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703125.842, "dur": 12.938,
    "args": {
      "External id": 2465794,"Sequence number": 24740649, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10393
    }
  },
  {
    "ph": "s", "id": 2, "pid": 1336756, "tid": 1336756, "ts": 1514656703125.842,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703242.961, "dur": 40.595,
    "args": {
      "External id": 2465795,"Record function id": 0, "Ev Idx": 10394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703389.746, "dur": 36.510,
    "args": {
      "External id": 2465796,"Sequence number": 24740650, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10395
    }
  },
  {
    "ph": "s", "id": 1, "pid": 1336756, "tid": 1336756, "ts": 1514656703389.746,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ones_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703485.185, "dur": 25.922,
    "args": {
      "External id": 2465797,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703486.771, "dur": 8.739,
    "args": {
      "External id": 2465798,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703490.890, "dur": 3.996,
    "args": {
      "External id": 2465799,"Record function id": 0, "Concrete Inputs": ["[1]", "[1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514656703496.916, "dur": 13.845,
    "args": {
      "External id": 2465800,"Record function id": 0, "Concrete Inputs": ["", "1."], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336756, "tid": 1336756,
    "ts": 1514658027379.251, "dur": 85.207,
    "args": {
      "External id": 2465801,"Sequence number": 24740651, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336756, "tid": 1336756,
    "ts": 1514658027476.701, "dur": 24.764,
    "args": {
      "External id": 2465802,"Sequence number": 24740652, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514658027510.195, "dur": 51.458,
    "args": {
      "External id": 2465803,"Sequence number": 24740653, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514658028117.304, "dur": 31.299,
    "args": {
      "External id": 2465804,"Sequence number": 24740654, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514658028155.517, "dur": 12.479,
    "args": {
      "External id": 2465805,"Sequence number": 24740655, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336756, "tid": 1336756,
    "ts": 1514658030652.767, "dur": 3035.394,
    "args": {
      "External id": 2465806,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336756, "tid": 1336756,
    "ts": 1514658031320.581, "dur": 998.039,
    "args": {
      "External id": 2465807,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336756, "tid": 1336756,
    "ts": 1514658031341.439, "dur": 69.880,
    "args": {
      "External id": 2465808,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 10407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514658031345.121, "dur": 13.407,
    "args": {
      "External id": 2465809,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658031361.430, "dur": 49.580,
    "args": {
      "External id": 2465810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[36375]], "Ev Idx": 10409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658031363.759, "dur": 46.690,
    "args": {
      "External id": 2465811,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[36375], []], "Ev Idx": 10410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033719.146, "dur": 3.698,
    "args": {
      "External id": 2465812,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033725.249, "dur": 0.308,
    "args": {
      "External id": 2465813,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033726.813, "dur": 0.436,
    "args": {
      "External id": 2465814,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033728.856, "dur": 0.308,
    "args": {
      "External id": 2465815,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033730.219, "dur": 0.439,
    "args": {
      "External id": 2465816,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033735.295, "dur": 0.343,
    "args": {
      "External id": 2465817,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033736.751, "dur": 0.290,
    "args": {
      "External id": 2465818,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033740.133, "dur": 0.337,
    "args": {
      "External id": 2465819,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033741.547, "dur": 0.399,
    "args": {
      "External id": 2465820,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033743.126, "dur": 0.197,
    "args": {
      "External id": 2465821,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033744.351, "dur": 0.301,
    "args": {
      "External id": 2465822,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033745.592, "dur": 0.368,
    "args": {
      "External id": 2465823,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033747.012, "dur": 0.534,
    "args": {
      "External id": 2465824,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033748.413, "dur": 0.398,
    "args": {
      "External id": 2465825,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033749.731, "dur": 0.491,
    "args": {
      "External id": 2465826,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033752.577, "dur": 0.408,
    "args": {
      "External id": 2465827,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033753.824, "dur": 0.400,
    "args": {
      "External id": 2465828,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033755.243, "dur": 0.243,
    "args": {
      "External id": 2465829,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033756.269, "dur": 0.234,
    "args": {
      "External id": 2465830,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033757.378, "dur": 0.238,
    "args": {
      "External id": 2465831,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033758.377, "dur": 0.420,
    "args": {
      "External id": 2465832,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033759.637, "dur": 0.205,
    "args": {
      "External id": 2465833,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033760.610, "dur": 0.236,
    "args": {
      "External id": 2465834,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033763.102, "dur": 0.211,
    "args": {
      "External id": 2465835,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033764.239, "dur": 0.241,
    "args": {
      "External id": 2465836,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033765.451, "dur": 0.211,
    "args": {
      "External id": 2465837,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033766.439, "dur": 0.214,
    "args": {
      "External id": 2465838,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033767.442, "dur": 0.203,
    "args": {
      "External id": 2465839,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033768.404, "dur": 0.208,
    "args": {
      "External id": 2465840,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033769.395, "dur": 0.205,
    "args": {
      "External id": 2465841,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033770.646, "dur": 0.213,
    "args": {
      "External id": 2465842,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033773.504, "dur": 0.211,
    "args": {
      "External id": 2465843,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033774.691, "dur": 0.209,
    "args": {
      "External id": 2465844,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033776.080, "dur": 0.206,
    "args": {
      "External id": 2465845,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033777.410, "dur": 0.207,
    "args": {
      "External id": 2465846,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033778.392, "dur": 0.228,
    "args": {
      "External id": 2465847,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033779.381, "dur": 0.212,
    "args": {
      "External id": 2465848,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033780.488, "dur": 0.204,
    "args": {
      "External id": 2465849,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033781.465, "dur": 0.235,
    "args": {
      "External id": 2465850,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033784.635, "dur": 0.231,
    "args": {
      "External id": 2465851,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033785.797, "dur": 0.229,
    "args": {
      "External id": 2465852,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033787.071, "dur": 0.227,
    "args": {
      "External id": 2465853,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033788.292, "dur": 0.231,
    "args": {
      "External id": 2465854,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033789.460, "dur": 0.201,
    "args": {
      "External id": 2465855,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033790.571, "dur": 0.208,
    "args": {
      "External id": 2465856,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033791.558, "dur": 0.202,
    "args": {
      "External id": 2465857,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033792.794, "dur": 0.212,
    "args": {
      "External id": 2465858,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033795.507, "dur": 0.200,
    "args": {
      "External id": 2465859,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033796.586, "dur": 0.196,
    "args": {
      "External id": 2465860,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033797.728, "dur": 0.215,
    "args": {
      "External id": 2465861,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033802.571, "dur": 0.240,
    "args": {
      "External id": 2465862,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033803.622, "dur": 0.199,
    "args": {
      "External id": 2465863,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033804.621, "dur": 0.205,
    "args": {
      "External id": 2465864,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033806.121, "dur": 0.241,
    "args": {
      "External id": 2465865,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033807.166, "dur": 0.214,
    "args": {
      "External id": 2465866,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033809.845, "dur": 0.443,
    "args": {
      "External id": 2465867,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033811.200, "dur": 0.231,
    "args": {
      "External id": 2465868,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033812.244, "dur": 0.200,
    "args": {
      "External id": 2465869,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033813.242, "dur": 0.209,
    "args": {
      "External id": 2465870,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033814.326, "dur": 0.231,
    "args": {
      "External id": 2465871,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033815.419, "dur": 0.312,
    "args": {
      "External id": 2465872,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033816.957, "dur": 0.413,
    "args": {
      "External id": 2465873,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033818.319, "dur": 0.339,
    "args": {
      "External id": 2465874,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033821.313, "dur": 0.431,
    "args": {
      "External id": 2465875,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033822.733, "dur": 0.331,
    "args": {
      "External id": 2465876,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033824.348, "dur": 0.200,
    "args": {
      "External id": 2465877,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033825.716, "dur": 0.190,
    "args": {
      "External id": 2465878,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033827.091, "dur": 0.168,
    "args": {
      "External id": 2465879,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033828.055, "dur": 0.335,
    "args": {
      "External id": 2465880,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033829.296, "dur": 0.365,
    "args": {
      "External id": 2465881,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033830.452, "dur": 0.332,
    "args": {
      "External id": 2465882,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033833.058, "dur": 0.303,
    "args": {
      "External id": 2465883,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033834.152, "dur": 0.328,
    "args": {
      "External id": 2465884,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033835.468, "dur": 0.201,
    "args": {
      "External id": 2465885,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033836.652, "dur": 0.280,
    "args": {
      "External id": 2465886,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033838.006, "dur": 0.201,
    "args": {
      "External id": 2465887,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033839.176, "dur": 0.199,
    "args": {
      "External id": 2465888,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033841.118, "dur": 0.203,
    "args": {
      "External id": 2465889,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033842.448, "dur": 0.220,
    "args": {
      "External id": 2465890,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033845.379, "dur": 0.230,
    "args": {
      "External id": 2465891,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033846.403, "dur": 0.206,
    "args": {
      "External id": 2465892,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033847.612, "dur": 0.198,
    "args": {
      "External id": 2465893,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033848.581, "dur": 0.204,
    "args": {
      "External id": 2465894,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033849.684, "dur": 0.200,
    "args": {
      "External id": 2465895,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033850.661, "dur": 0.204,
    "args": {
      "External id": 2465896,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033851.680, "dur": 0.201,
    "args": {
      "External id": 2465897,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033852.702, "dur": 0.204,
    "args": {
      "External id": 2465898,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033855.339, "dur": 0.199,
    "args": {
      "External id": 2465899,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033856.372, "dur": 0.204,
    "args": {
      "External id": 2465900,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033857.348, "dur": 0.197,
    "args": {
      "External id": 2465901,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033858.334, "dur": 0.205,
    "args": {
      "External id": 2465902,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033859.311, "dur": 0.197,
    "args": {
      "External id": 2465903,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033860.275, "dur": 0.215,
    "args": {
      "External id": 2465904,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033862.346, "dur": 0.205,
    "args": {
      "External id": 2465905,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033863.441, "dur": 0.513,
    "args": {
      "External id": 2465906,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033866.811, "dur": 0.413,
    "args": {
      "External id": 2465907,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033868.165, "dur": 0.438,
    "args": {
      "External id": 2465908,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033869.408, "dur": 0.202,
    "args": {
      "External id": 2465909,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033870.375, "dur": 0.202,
    "args": {
      "External id": 2465910,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033871.350, "dur": 0.422,
    "args": {
      "External id": 2465911,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033872.642, "dur": 0.205,
    "args": {
      "External id": 2465912,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033873.820, "dur": 0.200,
    "args": {
      "External id": 2465913,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033874.975, "dur": 0.206,
    "args": {
      "External id": 2465914,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033877.392, "dur": 0.250,
    "args": {
      "External id": 2465915,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033878.412, "dur": 0.205,
    "args": {
      "External id": 2465916,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033879.515, "dur": 0.202,
    "args": {
      "External id": 2465917,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033880.521, "dur": 0.265,
    "args": {
      "External id": 2465918,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033881.687, "dur": 0.275,
    "args": {
      "External id": 2465919,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033882.801, "dur": 0.358,
    "args": {
      "External id": 2465920,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033884.394, "dur": 0.350,
    "args": {
      "External id": 2465921,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033885.722, "dur": 0.308,
    "args": {
      "External id": 2465922,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033888.685, "dur": 0.206,
    "args": {
      "External id": 2465923,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033889.905, "dur": 0.303,
    "args": {
      "External id": 2465924,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033891.279, "dur": 0.199,
    "args": {
      "External id": 2465925,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033892.397, "dur": 0.335,
    "args": {
      "External id": 2465926,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033893.868, "dur": 0.341,
    "args": {
      "External id": 2465927,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033895.124, "dur": 0.305,
    "args": {
      "External id": 2465928,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033896.424, "dur": 0.208,
    "args": {
      "External id": 2465929,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033897.659, "dur": 0.202,
    "args": {
      "External id": 2465930,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033900.247, "dur": 0.203,
    "args": {
      "External id": 2465931,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033901.423, "dur": 0.201,
    "args": {
      "External id": 2465932,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033902.398, "dur": 0.198,
    "args": {
      "External id": 2465933,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033903.361, "dur": 0.201,
    "args": {
      "External id": 2465934,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033904.443, "dur": 0.197,
    "args": {
      "External id": 2465935,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033905.448, "dur": 0.237,
    "args": {
      "External id": 2465936,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033906.522, "dur": 0.199,
    "args": {
      "External id": 2465937,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033907.583, "dur": 0.234,
    "args": {
      "External id": 2465938,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033910.312, "dur": 0.231,
    "args": {
      "External id": 2465939,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033911.322, "dur": 0.403,
    "args": {
      "External id": 2465940,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033912.930, "dur": 0.196,
    "args": {
      "External id": 2465941,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033914.012, "dur": 0.201,
    "args": {
      "External id": 2465942,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033914.994, "dur": 0.193,
    "args": {
      "External id": 2465943,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033915.988, "dur": 0.207,
    "args": {
      "External id": 2465944,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033917.023, "dur": 0.209,
    "args": {
      "External id": 2465945,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033918.046, "dur": 0.206,
    "args": {
      "External id": 2465946,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033920.343, "dur": 0.200,
    "args": {
      "External id": 2465947,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033921.321, "dur": 0.622,
    "args": {
      "External id": 2465948,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033922.741, "dur": 0.200,
    "args": {
      "External id": 2465949,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033923.932, "dur": 0.174,
    "args": {
      "External id": 2465950,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033925.014, "dur": 0.292,
    "args": {
      "External id": 2465951,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033926.082, "dur": 0.326,
    "args": {
      "External id": 2465952,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033927.203, "dur": 0.500,
    "args": {
      "External id": 2465953,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033928.492, "dur": 0.389,
    "args": {
      "External id": 2465954,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033931.586, "dur": 0.318,
    "args": {
      "External id": 2465955,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033932.730, "dur": 0.514,
    "args": {
      "External id": 2465956,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033934.096, "dur": 0.203,
    "args": {
      "External id": 2465957,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033935.078, "dur": 0.202,
    "args": {
      "External id": 2465958,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033936.071, "dur": 0.199,
    "args": {
      "External id": 2465959,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033937.037, "dur": 0.202,
    "args": {
      "External id": 2465960,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033938.433, "dur": 0.206,
    "args": {
      "External id": 2465961,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033939.416, "dur": 0.210,
    "args": {
      "External id": 2465962,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033941.877, "dur": 0.202,
    "args": {
      "External id": 2465963,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033942.916, "dur": 0.204,
    "args": {
      "External id": 2465964,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033944.061, "dur": 0.219,
    "args": {
      "External id": 2465965,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033945.146, "dur": 0.226,
    "args": {
      "External id": 2465966,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033946.238, "dur": 0.189,
    "args": {
      "External id": 2465967,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033947.236, "dur": 0.204,
    "args": {
      "External id": 2465968,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033948.394, "dur": 0.205,
    "args": {
      "External id": 2465969,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033949.556, "dur": 0.206,
    "args": {
      "External id": 2465970,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033952.192, "dur": 0.233,
    "args": {
      "External id": 2465971,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033953.355, "dur": 0.205,
    "args": {
      "External id": 2465972,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033954.514, "dur": 0.200,
    "args": {
      "External id": 2465973,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033955.677, "dur": 0.246,
    "args": {
      "External id": 2465974,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033956.905, "dur": 0.200,
    "args": {
      "External id": 2465975,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033958.939, "dur": 0.203,
    "args": {
      "External id": 2465976,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033959.996, "dur": 0.201,
    "args": {
      "External id": 2465977,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033961.079, "dur": 0.204,
    "args": {
      "External id": 2465978,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033963.852, "dur": 0.202,
    "args": {
      "External id": 2465979,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033964.912, "dur": 0.234,
    "args": {
      "External id": 2465980,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033965.926, "dur": 0.200,
    "args": {
      "External id": 2465981,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033966.898, "dur": 0.202,
    "args": {
      "External id": 2465982,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033967.910, "dur": 0.200,
    "args": {
      "External id": 2465983,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033968.905, "dur": 0.203,
    "args": {
      "External id": 2465984,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033970.183, "dur": 0.203,
    "args": {
      "External id": 2465985,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033971.167, "dur": 0.209,
    "args": {
      "External id": 2465986,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033973.539, "dur": 0.198,
    "args": {
      "External id": 2465987,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033974.516, "dur": 0.221,
    "args": {
      "External id": 2465988,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033975.568, "dur": 0.198,
    "args": {
      "External id": 2465989,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033976.592, "dur": 0.204,
    "args": {
      "External id": 2465990,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033977.575, "dur": 0.229,
    "args": {
      "External id": 2465991,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033978.723, "dur": 0.204,
    "args": {
      "External id": 2465992,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033979.721, "dur": 0.199,
    "args": {
      "External id": 2465993,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658033980.720, "dur": 0.204,
    "args": {
      "External id": 2465994,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034008.657, "dur": 0.960,
    "args": {
      "External id": 2465995,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034013.185, "dur": 0.614,
    "args": {
      "External id": 2465996,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034014.693, "dur": 0.245,
    "args": {
      "External id": 2465997,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034015.823, "dur": 0.400,
    "args": {
      "External id": 2465998,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034016.990, "dur": 0.453,
    "args": {
      "External id": 2465999,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034018.220, "dur": 0.423,
    "args": {
      "External id": 2466000,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034019.653, "dur": 0.601,
    "args": {
      "External id": 2466001,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034021.208, "dur": 0.440,
    "args": {
      "External id": 2466002,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034023.767, "dur": 0.535,
    "args": {
      "External id": 2466003,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034025.278, "dur": 0.402,
    "args": {
      "External id": 2466004,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034026.684, "dur": 0.219,
    "args": {
      "External id": 2466005,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034027.810, "dur": 0.310,
    "args": {
      "External id": 2466006,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034029.198, "dur": 0.206,
    "args": {
      "External id": 2466007,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034030.165, "dur": 0.198,
    "args": {
      "External id": 2466008,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034031.114, "dur": 0.208,
    "args": {
      "External id": 2466009,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034032.126, "dur": 0.203,
    "args": {
      "External id": 2466010,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034034.565, "dur": 0.215,
    "args": {
      "External id": 2466011,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034035.576, "dur": 0.211,
    "args": {
      "External id": 2466012,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034037.369, "dur": 0.209,
    "args": {
      "External id": 2466013,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034038.375, "dur": 0.200,
    "args": {
      "External id": 2466014,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034039.475, "dur": 0.407,
    "args": {
      "External id": 2466015,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034040.973, "dur": 0.443,
    "args": {
      "External id": 2466016,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034042.338, "dur": 0.445,
    "args": {
      "External id": 2466017,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034043.786, "dur": 0.406,
    "args": {
      "External id": 2466018,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034046.714, "dur": 0.395,
    "args": {
      "External id": 2466019,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034047.929, "dur": 0.464,
    "args": {
      "External id": 2466020,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034049.271, "dur": 0.206,
    "args": {
      "External id": 2466021,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034050.241, "dur": 0.382,
    "args": {
      "External id": 2466022,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034051.437, "dur": 0.304,
    "args": {
      "External id": 2466023,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034052.496, "dur": 0.328,
    "args": {
      "External id": 2466024,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034053.614, "dur": 0.330,
    "args": {
      "External id": 2466025,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034054.709, "dur": 0.350,
    "args": {
      "External id": 2466026,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034057.574, "dur": 0.328,
    "args": {
      "External id": 2466027,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034058.836, "dur": 0.446,
    "args": {
      "External id": 2466028,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034060.121, "dur": 0.203,
    "args": {
      "External id": 2466029,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034061.137, "dur": 0.340,
    "args": {
      "External id": 2466030,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034062.247, "dur": 0.204,
    "args": {
      "External id": 2466031,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034063.269, "dur": 0.198,
    "args": {
      "External id": 2466032,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034064.248, "dur": 0.207,
    "args": {
      "External id": 2466033,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034065.274, "dur": 0.202,
    "args": {
      "External id": 2466034,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034067.614, "dur": 0.207,
    "args": {
      "External id": 2466035,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034068.614, "dur": 0.193,
    "args": {
      "External id": 2466036,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034069.557, "dur": 0.197,
    "args": {
      "External id": 2466037,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034070.566, "dur": 0.234,
    "args": {
      "External id": 2466038,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034071.590, "dur": 0.200,
    "args": {
      "External id": 2466039,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034072.546, "dur": 0.199,
    "args": {
      "External id": 2466040,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034073.511, "dur": 0.202,
    "args": {
      "External id": 2466041,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034074.475, "dur": 0.209,
    "args": {
      "External id": 2466042,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034076.854, "dur": 0.203,
    "args": {
      "External id": 2466043,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034077.853, "dur": 0.194,
    "args": {
      "External id": 2466044,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034079.312, "dur": 0.243,
    "args": {
      "External id": 2466045,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034080.397, "dur": 0.201,
    "args": {
      "External id": 2466046,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034081.391, "dur": 0.200,
    "args": {
      "External id": 2466047,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034082.348, "dur": 0.197,
    "args": {
      "External id": 2466048,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034083.373, "dur": 0.204,
    "args": {
      "External id": 2466049,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034084.553, "dur": 0.192,
    "args": {
      "External id": 2466050,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034086.940, "dur": 0.200,
    "args": {
      "External id": 2466051,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034088.139, "dur": 0.226,
    "args": {
      "External id": 2466052,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034089.426, "dur": 0.205,
    "args": {
      "External id": 2466053,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034090.592, "dur": 0.457,
    "args": {
      "External id": 2466054,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034091.821, "dur": 0.205,
    "args": {
      "External id": 2466055,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034093.066, "dur": 0.198,
    "args": {
      "External id": 2466056,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034094.204, "dur": 0.480,
    "args": {
      "External id": 2466057,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034095.628, "dur": 0.198,
    "args": {
      "External id": 2466058,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034098.083, "dur": 0.454,
    "args": {
      "External id": 2466059,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034099.529, "dur": 0.345,
    "args": {
      "External id": 2466060,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034100.772, "dur": 0.206,
    "args": {
      "External id": 2466061,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034101.845, "dur": 0.202,
    "args": {
      "External id": 2466062,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034102.966, "dur": 0.200,
    "args": {
      "External id": 2466063,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034104.024, "dur": 0.202,
    "args": {
      "External id": 2466064,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034105.112, "dur": 0.211,
    "args": {
      "External id": 2466065,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034106.081, "dur": 0.202,
    "args": {
      "External id": 2466066,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034108.478, "dur": 0.204,
    "args": {
      "External id": 2466067,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034109.445, "dur": 0.201,
    "args": {
      "External id": 2466068,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034110.395, "dur": 0.197,
    "args": {
      "External id": 2466069,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034111.373, "dur": 0.209,
    "args": {
      "External id": 2466070,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034112.468, "dur": 0.196,
    "args": {
      "External id": 2466071,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034113.635, "dur": 0.231,
    "args": {
      "External id": 2466072,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034117.415, "dur": 0.216,
    "args": {
      "External id": 2466073,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034118.412, "dur": 0.207,
    "args": {
      "External id": 2466074,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034120.655, "dur": 0.203,
    "args": {
      "External id": 2466075,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034121.657, "dur": 0.369,
    "args": {
      "External id": 2466076,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034122.832, "dur": 0.204,
    "args": {
      "External id": 2466077,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034123.921, "dur": 0.327,
    "args": {
      "External id": 2466078,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034125.094, "dur": 0.204,
    "args": {
      "External id": 2466079,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034126.138, "dur": 0.240,
    "args": {
      "External id": 2466080,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034129.709, "dur": 0.325,
    "args": {
      "External id": 2466081,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034130.820, "dur": 0.199,
    "args": {
      "External id": 2466082,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034133.179, "dur": 0.310,
    "args": {
      "External id": 2466083,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034134.275, "dur": 0.189,
    "args": {
      "External id": 2466084,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034135.244, "dur": 0.200,
    "args": {
      "External id": 2466085,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034136.214, "dur": 0.197,
    "args": {
      "External id": 2466086,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034137.180, "dur": 0.417,
    "args": {
      "External id": 2466087,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034138.354, "dur": 0.417,
    "args": {
      "External id": 2466088,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034140.659, "dur": 0.245,
    "args": {
      "External id": 2466089,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034141.856, "dur": 0.218,
    "args": {
      "External id": 2466090,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034144.330, "dur": 0.210,
    "args": {
      "External id": 2466091,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034145.484, "dur": 0.201,
    "args": {
      "External id": 2466092,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034146.433, "dur": 0.209,
    "args": {
      "External id": 2466093,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034147.396, "dur": 0.204,
    "args": {
      "External id": 2466094,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034148.348, "dur": 0.371,
    "args": {
      "External id": 2466095,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034149.487, "dur": 0.202,
    "args": {
      "External id": 2466096,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034150.481, "dur": 0.206,
    "args": {
      "External id": 2466097,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034151.813, "dur": 0.204,
    "args": {
      "External id": 2466098,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034154.557, "dur": 0.209,
    "args": {
      "External id": 2466099,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034155.546, "dur": 0.201,
    "args": {
      "External id": 2466100,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034156.587, "dur": 0.200,
    "args": {
      "External id": 2466101,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034157.603, "dur": 0.199,
    "args": {
      "External id": 2466102,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034214.360, "dur": 1514.502,
    "args": {
      "External id": 2466103,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034664.122, "dur": 988.726,
    "args": {
      "External id": 2466104,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034673.489, "dur": 7.455,
    "args": {
      "External id": 2466105,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034677.179, "dur": 3.208,
    "args": {
      "External id": 2466106,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034681.703, "dur": 3.411,
    "args": {
      "External id": 2466107,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034682.697, "dur": 2.287,
    "args": {
      "External id": 2466108,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034685.743, "dur": 2.762,
    "args": {
      "External id": 2466109,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034687.806, "dur": 0.619,
    "args": {
      "External id": 2466110,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034688.933, "dur": 1.671,
    "args": {
      "External id": 2466111,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034689.722, "dur": 0.806,
    "args": {
      "External id": 2466112,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034692.715, "dur": 4.076,
    "args": {
      "External id": 2466113,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034696.323, "dur": 0.395,
    "args": {
      "External id": 2466114,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034697.071, "dur": 1.472,
    "args": {
      "External id": 2466115,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034697.750, "dur": 0.726,
    "args": {
      "External id": 2466116,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034699.010, "dur": 1.074,
    "args": {
      "External id": 2466117,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034699.596, "dur": 0.414,
    "args": {
      "External id": 2466118,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034700.490, "dur": 3.557,
    "args": {
      "External id": 2466119,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034703.151, "dur": 0.714,
    "args": {
      "External id": 2466120,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034704.474, "dur": 1.523,
    "args": {
      "External id": 2466121,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034705.314, "dur": 0.616,
    "args": {
      "External id": 2466122,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034706.224, "dur": 3.123,
    "args": {
      "External id": 2466123,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034706.869, "dur": 2.398,
    "args": {
      "External id": 2466124,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034709.609, "dur": 2.173,
    "args": {
      "External id": 2466125,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034711.189, "dur": 0.528,
    "args": {
      "External id": 2466126,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034712.029, "dur": 1.084,
    "args": {
      "External id": 2466127,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034712.400, "dur": 0.542,
    "args": {
      "External id": 2466128,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034714.800, "dur": 3.934,
    "args": {
      "External id": 2466129,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034718.160, "dur": 0.505,
    "args": {
      "External id": 2466130,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034719.175, "dur": 1.660,
    "args": {
      "External id": 2466131,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034720.042, "dur": 0.718,
    "args": {
      "External id": 2466132,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034721.334, "dur": 1.164,
    "args": {
      "External id": 2466133,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034721.915, "dur": 0.512,
    "args": {
      "External id": 2466134,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034722.895, "dur": 2.579,
    "args": {
      "External id": 2466135,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034724.860, "dur": 0.547,
    "args": {
      "External id": 2466136,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034725.876, "dur": 1.211,
    "args": {
      "External id": 2466137,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034726.442, "dur": 0.576,
    "args": {
      "External id": 2466138,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034727.492, "dur": 2.854,
    "args": {
      "External id": 2466139,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034728.081, "dur": 2.193,
    "args": {
      "External id": 2466140,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034730.769, "dur": 2.009,
    "args": {
      "External id": 2466141,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034732.164, "dur": 0.547,
    "args": {
      "External id": 2466142,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034733.216, "dur": 1.338,
    "args": {
      "External id": 2466143,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034733.806, "dur": 0.680,
    "args": {
      "External id": 2466144,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034736.501, "dur": 4.069,
    "args": {
      "External id": 2466145,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034739.701, "dur": 0.797,
    "args": {
      "External id": 2466146,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034740.845, "dur": 1.591,
    "args": {
      "External id": 2466147,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034741.714, "dur": 0.652,
    "args": {
      "External id": 2466148,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034742.880, "dur": 1.253,
    "args": {
      "External id": 2466149,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034743.427, "dur": 0.540,
    "args": {
      "External id": 2466150,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034744.548, "dur": 2.732,
    "args": {
      "External id": 2466151,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034746.786, "dur": 0.426,
    "args": {
      "External id": 2466152,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034747.691, "dur": 1.107,
    "args": {
      "External id": 2466153,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034748.294, "dur": 0.435,
    "args": {
      "External id": 2466154,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034749.022, "dur": 2.420,
    "args": {
      "External id": 2466155,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034749.445, "dur": 1.920,
    "args": {
      "External id": 2466156,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034751.670, "dur": 2.016,
    "args": {
      "External id": 2466157,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034753.114, "dur": 0.501,
    "args": {
      "External id": 2466158,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034753.989, "dur": 1.016,
    "args": {
      "External id": 2466159,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034754.358, "dur": 0.574,
    "args": {
      "External id": 2466160,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034756.810, "dur": 3.881,
    "args": {
      "External id": 2466161,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034759.894, "dur": 0.731,
    "args": {
      "External id": 2466162,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034761.213, "dur": 1.736,
    "args": {
      "External id": 2466163,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034762.078, "dur": 0.807,
    "args": {
      "External id": 2466164,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034763.355, "dur": 1.394,
    "args": {
      "External id": 2466165,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034763.951, "dur": 0.729,
    "args": {
      "External id": 2466166,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034765.198, "dur": 3.128,
    "args": {
      "External id": 2466167,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034767.720, "dur": 0.446,
    "args": {
      "External id": 2466168,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034768.753, "dur": 1.717,
    "args": {
      "External id": 2466169,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034769.566, "dur": 0.560,
    "args": {
      "External id": 2466170,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034770.883, "dur": 2.755,
    "args": {
      "External id": 2466171,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034771.676, "dur": 1.885,
    "args": {
      "External id": 2466172,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034774.101, "dur": 2.630,
    "args": {
      "External id": 2466173,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034776.007, "dur": 0.464,
    "args": {
      "External id": 2466174,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034776.961, "dur": 1.301,
    "args": {
      "External id": 2466175,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034777.331, "dur": 0.767,
    "args": {
      "External id": 2466176,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034779.971, "dur": 3.834,
    "args": {
      "External id": 2466177,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034782.921, "dur": 0.813,
    "args": {
      "External id": 2466178,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034784.085, "dur": 1.597,
    "args": {
      "External id": 2466179,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034784.850, "dur": 0.762,
    "args": {
      "External id": 2466180,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034785.901, "dur": 1.141,
    "args": {
      "External id": 2466181,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034786.274, "dur": 0.697,
    "args": {
      "External id": 2466182,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034787.265, "dur": 4.335,
    "args": {
      "External id": 2466183,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034790.869, "dur": 0.664,
    "args": {
      "External id": 2466184,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034792.204, "dur": 1.623,
    "args": {
      "External id": 2466185,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034792.999, "dur": 0.662,
    "args": {
      "External id": 2466186,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034794.229, "dur": 2.689,
    "args": {
      "External id": 2466187,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034795.075, "dur": 1.676,
    "args": {
      "External id": 2466188,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034797.351, "dur": 2.461,
    "args": {
      "External id": 2466189,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034799.123, "dur": 0.622,
    "args": {
      "External id": 2466190,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034800.123, "dur": 1.228,
    "args": {
      "External id": 2466191,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034800.724, "dur": 0.560,
    "args": {
      "External id": 2466192,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034803.326, "dur": 4.127,
    "args": {
      "External id": 2466193,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034806.632, "dur": 0.755,
    "args": {
      "External id": 2466194,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034807.695, "dur": 1.402,
    "args": {
      "External id": 2466195,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034808.313, "dur": 0.718,
    "args": {
      "External id": 2466196,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034809.324, "dur": 1.357,
    "args": {
      "External id": 2466197,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034809.719, "dur": 0.891,
    "args": {
      "External id": 2466198,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034811.090, "dur": 3.483,
    "args": {
      "External id": 2466199,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034813.891, "dur": 0.615,
    "args": {
      "External id": 2466200,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034814.974, "dur": 2.116,
    "args": {
      "External id": 2466201,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034815.887, "dur": 0.941,
    "args": {
      "External id": 2466202,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034817.514, "dur": 3.115,
    "args": {
      "External id": 2466203,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034818.692, "dur": 1.861,
    "args": {
      "External id": 2466204,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034820.890, "dur": 1.474,
    "args": {
      "External id": 2466205,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034821.752, "dur": 0.541,
    "args": {
      "External id": 2466206,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034822.585, "dur": 1.612,
    "args": {
      "External id": 2466207,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034823.581, "dur": 0.545,
    "args": {
      "External id": 2466208,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034826.099, "dur": 2.748,
    "args": {
      "External id": 2466209,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034828.311, "dur": 0.468,
    "args": {
      "External id": 2466210,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034829.113, "dur": 1.299,
    "args": {
      "External id": 2466211,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034829.913, "dur": 0.434,
    "args": {
      "External id": 2466212,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034830.668, "dur": 1.418,
    "args": {
      "External id": 2466213,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034831.433, "dur": 0.582,
    "args": {
      "External id": 2466214,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034832.391, "dur": 3.646,
    "args": {
      "External id": 2466215,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034835.338, "dur": 0.633,
    "args": {
      "External id": 2466216,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034836.510, "dur": 2.153,
    "args": {
      "External id": 2466217,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034837.702, "dur": 0.706,
    "args": {
      "External id": 2466218,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034839.091, "dur": 3.109,
    "args": {
      "External id": 2466219,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034840.189, "dur": 1.935,
    "args": {
      "External id": 2466220,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034842.775, "dur": 1.504,
    "args": {
      "External id": 2466221,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034843.771, "dur": 0.441,
    "args": {
      "External id": 2466222,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034844.588, "dur": 1.841,
    "args": {
      "External id": 2466223,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034845.753, "dur": 0.605,
    "args": {
      "External id": 2466224,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034848.184, "dur": 3.418,
    "args": {
      "External id": 2466225,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034850.825, "dur": 0.700,
    "args": {
      "External id": 2466226,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034852.324, "dur": 1.855,
    "args": {
      "External id": 2466227,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034853.466, "dur": 0.647,
    "args": {
      "External id": 2466228,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034854.622, "dur": 2.169,
    "args": {
      "External id": 2466229,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034855.595, "dur": 0.951,
    "args": {
      "External id": 2466230,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034857.203, "dur": 3.651,
    "args": {
      "External id": 2466231,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034859.977, "dur": 0.809,
    "args": {
      "External id": 2466232,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034861.128, "dur": 1.885,
    "args": {
      "External id": 2466233,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034862.102, "dur": 0.690,
    "args": {
      "External id": 2466234,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034863.238, "dur": 3.245,
    "args": {
      "External id": 2466235,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034864.382, "dur": 2.023,
    "args": {
      "External id": 2466236,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034866.742, "dur": 1.566,
    "args": {
      "External id": 2466237,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034867.559, "dur": 0.683,
    "args": {
      "External id": 2466238,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034868.537, "dur": 1.838,
    "args": {
      "External id": 2466239,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034869.637, "dur": 0.560,
    "args": {
      "External id": 2466240,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034871.939, "dur": 3.422,
    "args": {
      "External id": 2466241,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034874.599, "dur": 0.695,
    "args": {
      "External id": 2466242,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034875.605, "dur": 2.003,
    "args": {
      "External id": 2466243,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034876.839, "dur": 0.695,
    "args": {
      "External id": 2466244,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034878.067, "dur": 1.841,
    "args": {
      "External id": 2466245,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034879.288, "dur": 0.548,
    "args": {
      "External id": 2466246,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034880.341, "dur": 3.133,
    "args": {
      "External id": 2466247,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034882.973, "dur": 0.435,
    "args": {
      "External id": 2466248,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034883.927, "dur": 1.801,
    "args": {
      "External id": 2466249,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034885.206, "dur": 0.455,
    "args": {
      "External id": 2466250,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034885.957, "dur": 3.442,
    "args": {
      "External id": 2466251,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034887.102, "dur": 2.222,
    "args": {
      "External id": 2466252,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034889.658, "dur": 1.255,
    "args": {
      "External id": 2466253,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034890.298, "dur": 0.546,
    "args": {
      "External id": 2466254,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034891.386, "dur": 1.878,
    "args": {
      "External id": 2466255,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034892.274, "dur": 0.736,
    "args": {
      "External id": 2466256,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034895.034, "dur": 3.044,
    "args": {
      "External id": 2466257,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034897.262, "dur": 0.745,
    "args": {
      "External id": 2466258,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034898.327, "dur": 2.508,
    "args": {
      "External id": 2466259,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034899.648, "dur": 1.113,
    "args": {
      "External id": 2466260,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034901.106, "dur": 1.556,
    "args": {
      "External id": 2466261,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034901.856, "dur": 0.647,
    "args": {
      "External id": 2466262,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034902.886, "dur": 3.494,
    "args": {
      "External id": 2466263,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034905.423, "dur": 0.741,
    "args": {
      "External id": 2466264,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034906.653, "dur": 1.736,
    "args": {
      "External id": 2466265,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034907.662, "dur": 0.652,
    "args": {
      "External id": 2466266,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034908.829, "dur": 3.355,
    "args": {
      "External id": 2466267,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034909.771, "dur": 2.171,
    "args": {
      "External id": 2466268,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034912.620, "dur": 1.430,
    "args": {
      "External id": 2466269,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034913.480, "dur": 0.500,
    "args": {
      "External id": 2466270,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034914.439, "dur": 1.557,
    "args": {
      "External id": 2466271,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034915.371, "dur": 0.551,
    "args": {
      "External id": 2466272,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034917.554, "dur": 3.291,
    "args": {
      "External id": 2466273,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034920.279, "dur": 0.500,
    "args": {
      "External id": 2466274,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034921.112, "dur": 1.801,
    "args": {
      "External id": 2466275,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034922.185, "dur": 0.661,
    "args": {
      "External id": 2466276,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034923.370, "dur": 1.515,
    "args": {
      "External id": 2466277,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034924.179, "dur": 0.636,
    "args": {
      "External id": 2466278,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034925.324, "dur": 3.513,
    "args": {
      "External id": 2466279,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034928.115, "dur": 0.647,
    "args": {
      "External id": 2466280,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034929.296, "dur": 2.379,
    "args": {
      "External id": 2466281,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034930.458, "dur": 1.060,
    "args": {
      "External id": 2466282,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034931.937, "dur": 3.151,
    "args": {
      "External id": 2466283,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034933.106, "dur": 1.905,
    "args": {
      "External id": 2466284,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034935.348, "dur": 1.681,
    "args": {
      "External id": 2466285,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034936.231, "dur": 0.734,
    "args": {
      "External id": 2466286,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034937.442, "dur": 2.066,
    "args": {
      "External id": 2466287,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034938.525, "dur": 0.825,
    "args": {
      "External id": 2466288,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034941.753, "dur": 2.619,
    "args": {
      "External id": 2466289,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034943.918, "dur": 0.376,
    "args": {
      "External id": 2466290,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034944.614, "dur": 1.618,
    "args": {
      "External id": 2466291,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034945.518, "dur": 0.648,
    "args": {
      "External id": 2466292,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034946.540, "dur": 1.469,
    "args": {
      "External id": 2466293,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034947.137, "dur": 0.712,
    "args": {
      "External id": 2466294,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034948.229, "dur": 3.041,
    "args": {
      "External id": 2466295,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034950.743, "dur": 0.458,
    "args": {
      "External id": 2466296,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034951.740, "dur": 1.753,
    "args": {
      "External id": 2466297,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034952.964, "dur": 0.463,
    "args": {
      "External id": 2466298,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034953.934, "dur": 3.100,
    "args": {
      "External id": 2466299,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034955.022, "dur": 1.843,
    "args": {
      "External id": 2466300,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034957.517, "dur": 1.484,
    "args": {
      "External id": 2466301,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034958.398, "dur": 0.539,
    "args": {
      "External id": 2466302,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034959.445, "dur": 1.944,
    "args": {
      "External id": 2466303,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658034960.757, "dur": 0.561,
    "args": {
      "External id": 2466304,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035010.676, "dur": 6.063,
    "args": {
      "External id": 2472449,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035015.592, "dur": 0.855,
    "args": {
      "External id": 2472450,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035017.187, "dur": 1.200,
    "args": {
      "External id": 2472451,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035017.985, "dur": 0.329,
    "args": {
      "External id": 2472452,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035018.676, "dur": 1.345,
    "args": {
      "External id": 2472453,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035019.302, "dur": 0.649,
    "args": {
      "External id": 2472454,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035020.271, "dur": 3.150,
    "args": {
      "External id": 2472455,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035022.672, "dur": 0.681,
    "args": {
      "External id": 2472456,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035023.674, "dur": 1.217,
    "args": {
      "External id": 2472457,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035024.296, "dur": 0.525,
    "args": {
      "External id": 2472458,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035025.135, "dur": 3.183,
    "args": {
      "External id": 2472459,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035026.017, "dur": 2.230,
    "args": {
      "External id": 2472460,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035028.638, "dur": 1.384,
    "args": {
      "External id": 2472461,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035029.384, "dur": 0.568,
    "args": {
      "External id": 2472462,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035030.385, "dur": 1.616,
    "args": {
      "External id": 2472463,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035031.358, "dur": 0.580,
    "args": {
      "External id": 2472464,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035033.952, "dur": 2.678,
    "args": {
      "External id": 2472465,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035036.113, "dur": 0.447,
    "args": {
      "External id": 2472466,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035036.916, "dur": 1.738,
    "args": {
      "External id": 2472467,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035038.058, "dur": 0.404,
    "args": {
      "External id": 2472468,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035038.918, "dur": 1.267,
    "args": {
      "External id": 2472469,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035039.593, "dur": 0.525,
    "args": {
      "External id": 2472470,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035040.448, "dur": 2.848,
    "args": {
      "External id": 2472471,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035042.721, "dur": 0.508,
    "args": {
      "External id": 2472472,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035043.555, "dur": 1.298,
    "args": {
      "External id": 2472473,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035044.309, "dur": 0.468,
    "args": {
      "External id": 2472474,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035045.109, "dur": 2.687,
    "args": {
      "External id": 2472475,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035046.052, "dur": 1.557,
    "args": {
      "External id": 2472476,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035048.047, "dur": 1.346,
    "args": {
      "External id": 2472477,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035048.701, "dur": 0.624,
    "args": {
      "External id": 2472478,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035049.720, "dur": 1.527,
    "args": {
      "External id": 2472479,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035050.592, "dur": 0.590,
    "args": {
      "External id": 2472480,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035052.877, "dur": 2.815,
    "args": {
      "External id": 2472481,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035055.126, "dur": 0.494,
    "args": {
      "External id": 2472482,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035055.938, "dur": 1.481,
    "args": {
      "External id": 2472483,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035056.841, "dur": 0.507,
    "args": {
      "External id": 2472484,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035057.721, "dur": 1.444,
    "args": {
      "External id": 2472485,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035058.373, "dur": 0.725,
    "args": {
      "External id": 2472486,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035059.416, "dur": 2.603,
    "args": {
      "External id": 2472487,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035061.556, "dur": 0.395,
    "args": {
      "External id": 2472488,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035062.265, "dur": 1.394,
    "args": {
      "External id": 2472489,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035062.953, "dur": 0.545,
    "args": {
      "External id": 2472490,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035063.899, "dur": 2.685,
    "args": {
      "External id": 2472491,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035064.823, "dur": 1.690,
    "args": {
      "External id": 2472492,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035066.871, "dur": 1.358,
    "args": {
      "External id": 2472493,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035067.558, "dur": 0.511,
    "args": {
      "External id": 2472494,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035068.580, "dur": 2.327,
    "args": {
      "External id": 2472495,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035070.078, "dur": 0.671,
    "args": {
      "External id": 2472496,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035072.694, "dur": 2.610,
    "args": {
      "External id": 2472497,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035074.802, "dur": 0.433,
    "args": {
      "External id": 2472498,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035075.570, "dur": 1.532,
    "args": {
      "External id": 2472499,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035076.634, "dur": 0.399,
    "args": {
      "External id": 2472500,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035077.379, "dur": 1.365,
    "args": {
      "External id": 2472501,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035077.974, "dur": 0.700,
    "args": {
      "External id": 2472502,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035079.124, "dur": 2.700,
    "args": {
      "External id": 2472503,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035081.307, "dur": 0.448,
    "args": {
      "External id": 2472504,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035082.083, "dur": 1.248,
    "args": {
      "External id": 2472505,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035082.792, "dur": 0.470,
    "args": {
      "External id": 2472506,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035083.572, "dur": 2.486,
    "args": {
      "External id": 2472507,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035084.583, "dur": 1.404,
    "args": {
      "External id": 2472508,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035086.328, "dur": 1.059,
    "args": {
      "External id": 2472509,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035086.885, "dur": 0.434,
    "args": {
      "External id": 2472510,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035087.609, "dur": 1.540,
    "args": {
      "External id": 2472511,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035088.566, "dur": 0.517,
    "args": {
      "External id": 2472512,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035090.865, "dur": 2.969,
    "args": {
      "External id": 2472513,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035093.303, "dur": 0.464,
    "args": {
      "External id": 2472514,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035094.108, "dur": 1.652,
    "args": {
      "External id": 2472515,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035095.225, "dur": 0.460,
    "args": {
      "External id": 2472516,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035096.020, "dur": 1.217,
    "args": {
      "External id": 2472517,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035096.560, "dur": 0.527,
    "args": {
      "External id": 2472518,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035097.607, "dur": 2.816,
    "args": {
      "External id": 2472519,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035099.940, "dur": 0.417,
    "args": {
      "External id": 2472520,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035100.663, "dur": 1.214,
    "args": {
      "External id": 2472521,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035101.370, "dur": 0.441,
    "args": {
      "External id": 2472522,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035102.111, "dur": 2.968,
    "args": {
      "External id": 2472523,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035103.108, "dur": 1.896,
    "args": {
      "External id": 2472524,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035105.321, "dur": 1.235,
    "args": {
      "External id": 2472525,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035106.000, "dur": 0.397,
    "args": {
      "External id": 2472526,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035106.909, "dur": 1.491,
    "args": {
      "External id": 2472527,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035107.825, "dur": 0.510,
    "args": {
      "External id": 2472528,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035110.130, "dur": 2.696,
    "args": {
      "External id": 2472529,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035112.385, "dur": 0.372,
    "args": {
      "External id": 2472530,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035113.096, "dur": 1.473,
    "args": {
      "External id": 2472531,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035113.969, "dur": 0.440,
    "args": {
      "External id": 2472532,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035114.829, "dur": 1.267,
    "args": {
      "External id": 2472533,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035115.493, "dur": 0.540,
    "args": {
      "External id": 2472534,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035116.419, "dur": 3.349,
    "args": {
      "External id": 2472535,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035119.082, "dur": 0.617,
    "args": {
      "External id": 2472536,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035120.004, "dur": 1.383,
    "args": {
      "External id": 2472537,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035120.781, "dur": 0.536,
    "args": {
      "External id": 2472538,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035121.622, "dur": 3.302,
    "args": {
      "External id": 2472539,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035122.736, "dur": 2.014,
    "args": {
      "External id": 2472540,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035125.205, "dur": 1.503,
    "args": {
      "External id": 2472541,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035125.917, "dur": 0.546,
    "args": {
      "External id": 2472542,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035127.162, "dur": 1.845,
    "args": {
      "External id": 2472543,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035128.173, "dur": 0.665,
    "args": {
      "External id": 2472544,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035130.985, "dur": 2.635,
    "args": {
      "External id": 2472545,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035132.997, "dur": 0.553,
    "args": {
      "External id": 2472546,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035133.861, "dur": 1.550,
    "args": {
      "External id": 2472547,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035134.657, "dur": 0.591,
    "args": {
      "External id": 2472548,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035135.657, "dur": 1.516,
    "args": {
      "External id": 2472549,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035136.439, "dur": 0.549,
    "args": {
      "External id": 2472550,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035137.511, "dur": 3.570,
    "args": {
      "External id": 2472551,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035140.312, "dur": 0.487,
    "args": {
      "External id": 2472552,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035141.339, "dur": 1.060,
    "args": {
      "External id": 2472553,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035141.881, "dur": 0.447,
    "args": {
      "External id": 2472554,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035142.690, "dur": 2.708,
    "args": {
      "External id": 2472555,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035143.651, "dur": 1.575,
    "args": {
      "External id": 2472556,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035145.647, "dur": 1.356,
    "args": {
      "External id": 2472557,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035146.247, "dur": 0.687,
    "args": {
      "External id": 2472558,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035147.247, "dur": 1.596,
    "args": {
      "External id": 2472559,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035148.091, "dur": 0.686,
    "args": {
      "External id": 2472560,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035150.613, "dur": 2.796,
    "args": {
      "External id": 2472561,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035152.867, "dur": 0.478,
    "args": {
      "External id": 2472562,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035153.659, "dur": 1.821,
    "args": {
      "External id": 2472563,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035154.628, "dur": 0.586,
    "args": {
      "External id": 2472564,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035155.740, "dur": 1.407,
    "args": {
      "External id": 2472565,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035156.397, "dur": 0.686,
    "args": {
      "External id": 2472566,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035157.371, "dur": 2.587,
    "args": {
      "External id": 2472567,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035159.418, "dur": 0.469,
    "args": {
      "External id": 2472568,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035160.195, "dur": 1.117,
    "args": {
      "External id": 2472569,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035160.791, "dur": 0.454,
    "args": {
      "External id": 2472570,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035161.536, "dur": 2.511,
    "args": {
      "External id": 2472571,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035162.487, "dur": 1.491,
    "args": {
      "External id": 2472572,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035164.334, "dur": 1.261,
    "args": {
      "External id": 2472573,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035164.940, "dur": 0.588,
    "args": {
      "External id": 2472574,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035165.917, "dur": 1.870,
    "args": {
      "External id": 2472575,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035166.888, "dur": 0.836,
    "args": {
      "External id": 2472576,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035169.473, "dur": 2.624,
    "args": {
      "External id": 2472577,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035171.583, "dur": 0.445,
    "args": {
      "External id": 2472578,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035172.382, "dur": 1.584,
    "args": {
      "External id": 2472579,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035173.458, "dur": 0.439,
    "args": {
      "External id": 2472580,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035174.206, "dur": 1.360,
    "args": {
      "External id": 2472581,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035174.867, "dur": 0.546,
    "args": {
      "External id": 2472582,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035175.786, "dur": 3.771,
    "args": {
      "External id": 2472583,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035179.053, "dur": 0.430,
    "args": {
      "External id": 2472584,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035179.794, "dur": 1.225,
    "args": {
      "External id": 2472585,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035180.418, "dur": 0.531,
    "args": {
      "External id": 2472586,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035181.277, "dur": 3.032,
    "args": {
      "External id": 2472587,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035182.375, "dur": 1.711,
    "args": {
      "External id": 2472588,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035184.618, "dur": 1.100,
    "args": {
      "External id": 2472589,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035185.219, "dur": 0.427,
    "args": {
      "External id": 2472590,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035186.044, "dur": 1.658,
    "args": {
      "External id": 2472591,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035187.076, "dur": 0.562,
    "args": {
      "External id": 2472592,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035189.757, "dur": 2.727,
    "args": {
      "External id": 2472593,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035192.017, "dur": 0.400,
    "args": {
      "External id": 2472594,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035192.732, "dur": 1.607,
    "args": {
      "External id": 2472595,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035193.720, "dur": 0.550,
    "args": {
      "External id": 2472596,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035194.572, "dur": 1.155,
    "args": {
      "External id": 2472597,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035195.150, "dur": 0.510,
    "args": {
      "External id": 2472598,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035195.952, "dur": 3.140,
    "args": {
      "External id": 2472599,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035198.565, "dur": 0.458,
    "args": {
      "External id": 2472600,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035199.337, "dur": 1.243,
    "args": {
      "External id": 2472601,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035199.986, "dur": 0.525,
    "args": {
      "External id": 2472602,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035200.820, "dur": 3.138,
    "args": {
      "External id": 2472603,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035201.888, "dur": 2.003,
    "args": {
      "External id": 2472604,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035204.201, "dur": 1.530,
    "args": {
      "External id": 2472605,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035204.812, "dur": 0.856,
    "args": {
      "External id": 2472606,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035205.972, "dur": 1.467,
    "args": {
      "External id": 2472607,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035206.776, "dur": 0.599,
    "args": {
      "External id": 2472608,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035209.257, "dur": 2.681,
    "args": {
      "External id": 2472609,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035211.405, "dur": 0.464,
    "args": {
      "External id": 2472610,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035212.179, "dur": 1.519,
    "args": {
      "External id": 2472611,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035213.117, "dur": 0.408,
    "args": {
      "External id": 2472612,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035213.968, "dur": 1.384,
    "args": {
      "External id": 2472613,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035214.722, "dur": 0.568,
    "args": {
      "External id": 2472614,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035215.575, "dur": 2.747,
    "args": {
      "External id": 2472615,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035217.621, "dur": 0.543,
    "args": {
      "External id": 2472616,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035218.558, "dur": 1.373,
    "args": {
      "External id": 2472617,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035219.298, "dur": 0.565,
    "args": {
      "External id": 2472618,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035220.177, "dur": 2.572,
    "args": {
      "External id": 2472619,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035221.146, "dur": 1.528,
    "args": {
      "External id": 2472620,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035222.991, "dur": 1.128,
    "args": {
      "External id": 2472621,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035223.544, "dur": 0.511,
    "args": {
      "External id": 2472622,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035224.346, "dur": 1.599,
    "args": {
      "External id": 2472623,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035225.287, "dur": 0.592,
    "args": {
      "External id": 2472624,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035228.099, "dur": 2.501,
    "args": {
      "External id": 2472625,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035230.118, "dur": 0.412,
    "args": {
      "External id": 2472626,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035230.838, "dur": 1.558,
    "args": {
      "External id": 2472627,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035231.731, "dur": 0.508,
    "args": {
      "External id": 2472628,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035232.633, "dur": 1.469,
    "args": {
      "External id": 2472629,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035233.428, "dur": 0.492,
    "args": {
      "External id": 2472630,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035234.460, "dur": 2.842,
    "args": {
      "External id": 2472631,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035236.770, "dur": 0.460,
    "args": {
      "External id": 2472632,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035237.540, "dur": 1.281,
    "args": {
      "External id": 2472633,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035238.346, "dur": 0.403,
    "args": {
      "External id": 2472634,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035239.041, "dur": 2.820,
    "args": {
      "External id": 2472635,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035240.039, "dur": 1.753,
    "args": {
      "External id": 2472636,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035242.107, "dur": 1.078,
    "args": {
      "External id": 2472637,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035242.678, "dur": 0.437,
    "args": {
      "External id": 2472638,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035243.494, "dur": 1.478,
    "args": {
      "External id": 2472639,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035244.418, "dur": 0.437,
    "args": {
      "External id": 2472640,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035247.036, "dur": 2.557,
    "args": {
      "External id": 2472641,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035249.016, "dur": 0.507,
    "args": {
      "External id": 2472642,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035249.835, "dur": 1.452,
    "args": {
      "External id": 2472643,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035250.750, "dur": 0.465,
    "args": {
      "External id": 2472644,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035251.522, "dur": 1.247,
    "args": {
      "External id": 2472645,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035252.185, "dur": 0.521,
    "args": {
      "External id": 2472646,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035252.991, "dur": 3.182,
    "args": {
      "External id": 2472647,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035255.646, "dur": 0.455,
    "args": {
      "External id": 2472648,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035256.410, "dur": 1.157,
    "args": {
      "External id": 2472649,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035256.989, "dur": 0.507,
    "args": {
      "External id": 2472650,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035257.784, "dur": 2.797,
    "args": {
      "External id": 2472651,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035258.586, "dur": 1.716,
    "args": {
      "External id": 2472652,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035260.826, "dur": 1.082,
    "args": {
      "External id": 2472653,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035261.412, "dur": 0.428,
    "args": {
      "External id": 2472654,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035262.127, "dur": 1.411,
    "args": {
      "External id": 2472655,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035262.934, "dur": 0.540,
    "args": {
      "External id": 2472656,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035265.304, "dur": 2.619,
    "args": {
      "External id": 2472657,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035267.428, "dur": 0.427,
    "args": {
      "External id": 2472658,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035268.170, "dur": 1.517,
    "args": {
      "External id": 2472659,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035269.149, "dur": 0.466,
    "args": {
      "External id": 2472660,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035269.951, "dur": 1.452,
    "args": {
      "External id": 2472661,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035270.518, "dur": 0.821,
    "args": {
      "External id": 2472662,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035271.625, "dur": 3.001,
    "args": {
      "External id": 2472663,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035273.802, "dur": 0.751,
    "args": {
      "External id": 2472664,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035274.862, "dur": 1.100,
    "args": {
      "External id": 2472665,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035275.432, "dur": 0.458,
    "args": {
      "External id": 2472666,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035276.202, "dur": 3.314,
    "args": {
      "External id": 2472667,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035277.083, "dur": 2.366,
    "args": {
      "External id": 2472668,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035279.761, "dur": 1.478,
    "args": {
      "External id": 2472669,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035280.741, "dur": 0.431,
    "args": {
      "External id": 2472670,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035281.632, "dur": 1.642,
    "args": {
      "External id": 2472671,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035282.652, "dur": 0.556,
    "args": {
      "External id": 2472672,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035285.008, "dur": 2.506,
    "args": {
      "External id": 2472673,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035287.022, "dur": 0.424,
    "args": {
      "External id": 2472674,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035287.761, "dur": 1.688,
    "args": {
      "External id": 2472675,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035288.741, "dur": 0.545,
    "args": {
      "External id": 2472676,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035289.688, "dur": 1.171,
    "args": {
      "External id": 2472677,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035290.270, "dur": 0.524,
    "args": {
      "External id": 2472678,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035291.078, "dur": 2.636,
    "args": {
      "External id": 2472679,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035293.253, "dur": 0.393,
    "args": {
      "External id": 2472680,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035293.969, "dur": 1.392,
    "args": {
      "External id": 2472681,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035294.685, "dur": 0.604,
    "args": {
      "External id": 2472682,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035295.581, "dur": 2.770,
    "args": {
      "External id": 2472683,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035296.407, "dur": 1.779,
    "args": {
      "External id": 2472684,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035298.596, "dur": 1.124,
    "args": {
      "External id": 2472685,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035299.254, "dur": 0.396,
    "args": {
      "External id": 2472686,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035299.964, "dur": 1.556,
    "args": {
      "External id": 2472687,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035300.909, "dur": 0.546,
    "args": {
      "External id": 2472688,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035303.173, "dur": 2.534,
    "args": {
      "External id": 2472689,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035305.245, "dur": 0.392,
    "args": {
      "External id": 2472690,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035305.946, "dur": 1.601,
    "args": {
      "External id": 2472691,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035306.920, "dur": 0.553,
    "args": {
      "External id": 2472692,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035307.972, "dur": 1.307,
    "args": {
      "External id": 2472693,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035308.553, "dur": 0.659,
    "args": {
      "External id": 2472694,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035309.496, "dur": 2.774,
    "args": {
      "External id": 2472695,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035311.770, "dur": 0.431,
    "args": {
      "External id": 2472696,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035312.535, "dur": 1.136,
    "args": {
      "External id": 2472697,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035313.183, "dur": 0.416,
    "args": {
      "External id": 2472698,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035313.892, "dur": 2.907,
    "args": {
      "External id": 2472699,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035315.019, "dur": 1.620,
    "args": {
      "External id": 2472700,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035317.039, "dur": 1.125,
    "args": {
      "External id": 2472701,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035317.669, "dur": 0.423,
    "args": {
      "External id": 2472702,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035318.664, "dur": 1.569,
    "args": {
      "External id": 2472703,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035319.475, "dur": 0.692,
    "args": {
      "External id": 2472704,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035322.211, "dur": 2.935,
    "args": {
      "External id": 2472705,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035324.465, "dur": 0.611,
    "args": {
      "External id": 2472706,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035325.413, "dur": 1.773,
    "args": {
      "External id": 2472707,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035326.410, "dur": 0.707,
    "args": {
      "External id": 2472708,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035327.426, "dur": 1.534,
    "args": {
      "External id": 2472709,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035328.141, "dur": 0.755,
    "args": {
      "External id": 2472710,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035329.265, "dur": 2.821,
    "args": {
      "External id": 2472711,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035331.581, "dur": 0.434,
    "args": {
      "External id": 2472712,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035332.339, "dur": 1.248,
    "args": {
      "External id": 2472713,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035333.050, "dur": 0.465,
    "args": {
      "External id": 2472714,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035333.825, "dur": 2.979,
    "args": {
      "External id": 2472715,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035334.672, "dur": 2.053,
    "args": {
      "External id": 2472716,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035337.072, "dur": 1.273,
    "args": {
      "External id": 2472717,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035337.761, "dur": 0.515,
    "args": {
      "External id": 2472718,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035338.902, "dur": 1.621,
    "args": {
      "External id": 2472719,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035339.819, "dur": 0.627,
    "args": {
      "External id": 2472720,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035342.320, "dur": 2.702,
    "args": {
      "External id": 2472721,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035344.461, "dur": 0.493,
    "args": {
      "External id": 2472722,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035345.264, "dur": 1.847,
    "args": {
      "External id": 2472723,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035346.267, "dur": 0.774,
    "args": {
      "External id": 2472724,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035347.356, "dur": 1.404,
    "args": {
      "External id": 2472725,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035348.146, "dur": 0.550,
    "args": {
      "External id": 2472726,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035349.000, "dur": 3.154,
    "args": {
      "External id": 2472727,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035351.545, "dur": 0.542,
    "args": {
      "External id": 2472728,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035352.528, "dur": 1.259,
    "args": {
      "External id": 2472729,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035353.119, "dur": 0.600,
    "args": {
      "External id": 2472730,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035354.007, "dur": 2.771,
    "args": {
      "External id": 2472731,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035354.830, "dur": 1.785,
    "args": {
      "External id": 2472732,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035357.038, "dur": 1.068,
    "args": {
      "External id": 2472733,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035357.604, "dur": 0.432,
    "args": {
      "External id": 2472734,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035358.325, "dur": 1.636,
    "args": {
      "External id": 2472735,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035359.363, "dur": 0.524,
    "args": {
      "External id": 2472736,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035362.287, "dur": 2.671,
    "args": {
      "External id": 2472737,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035364.536, "dur": 0.348,
    "args": {
      "External id": 2472738,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035365.261, "dur": 1.604,
    "args": {
      "External id": 2472739,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035366.331, "dur": 0.465,
    "args": {
      "External id": 2472740,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035367.124, "dur": 1.120,
    "args": {
      "External id": 2472741,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035367.732, "dur": 0.444,
    "args": {
      "External id": 2472742,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035368.561, "dur": 2.761,
    "args": {
      "External id": 2472743,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035370.776, "dur": 0.474,
    "args": {
      "External id": 2472744,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035371.560, "dur": 1.179,
    "args": {
      "External id": 2472745,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035372.201, "dur": 0.468,
    "args": {
      "External id": 2472746,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035372.960, "dur": 2.738,
    "args": {
      "External id": 2472747,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035373.968, "dur": 1.660,
    "args": {
      "External id": 2472748,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035375.941, "dur": 1.549,
    "args": {
      "External id": 2472749,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035376.737, "dur": 0.681,
    "args": {
      "External id": 2472750,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035377.711, "dur": 1.700,
    "args": {
      "External id": 2472751,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035378.505, "dur": 0.840,
    "args": {
      "External id": 2472752,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035380.979, "dur": 2.661,
    "args": {
      "External id": 2472753,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035383.153, "dur": 0.418,
    "args": {
      "External id": 2472754,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035383.903, "dur": 1.632,
    "args": {
      "External id": 2472755,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035384.794, "dur": 0.670,
    "args": {
      "External id": 2472756,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035385.793, "dur": 1.214,
    "args": {
      "External id": 2472757,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035386.377, "dur": 0.564,
    "args": {
      "External id": 2472758,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035387.350, "dur": 2.898,
    "args": {
      "External id": 2472759,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035389.699, "dur": 0.470,
    "args": {
      "External id": 2472760,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035390.807, "dur": 1.520,
    "args": {
      "External id": 2472761,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035391.778, "dur": 0.484,
    "args": {
      "External id": 2472762,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035392.546, "dur": 3.013,
    "args": {
      "External id": 2472763,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035393.492, "dur": 1.995,
    "args": {
      "External id": 2472764,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035395.830, "dur": 1.319,
    "args": {
      "External id": 2472765,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035396.427, "dur": 0.654,
    "args": {
      "External id": 2472766,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035397.567, "dur": 1.504,
    "args": {
      "External id": 2472767,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035398.495, "dur": 0.512,
    "args": {
      "External id": 2472768,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035400.801, "dur": 2.881,
    "args": {
      "External id": 2472769,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035403.150, "dur": 0.469,
    "args": {
      "External id": 2472770,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035403.922, "dur": 1.734,
    "args": {
      "External id": 2472771,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035405.115, "dur": 0.467,
    "args": {
      "External id": 2472772,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035405.894, "dur": 1.323,
    "args": {
      "External id": 2472773,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035406.594, "dur": 0.559,
    "args": {
      "External id": 2472774,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035407.768, "dur": 3.112,
    "args": {
      "External id": 2472775,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035410.376, "dur": 0.434,
    "args": {
      "External id": 2472776,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035411.313, "dur": 1.475,
    "args": {
      "External id": 2472777,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035412.082, "dur": 0.636,
    "args": {
      "External id": 2472778,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035413.005, "dur": 2.723,
    "args": {
      "External id": 2472779,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035414.115, "dur": 1.452,
    "args": {
      "External id": 2472780,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035415.971, "dur": 1.443,
    "args": {
      "External id": 2472781,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035416.566, "dur": 0.778,
    "args": {
      "External id": 2472782,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035417.761, "dur": 1.417,
    "args": {
      "External id": 2472783,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035418.567, "dur": 0.548,
    "args": {
      "External id": 2472784,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035420.889, "dur": 2.520,
    "args": {
      "External id": 2472785,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035422.850, "dur": 0.489,
    "args": {
      "External id": 2472786,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035423.647, "dur": 1.415,
    "args": {
      "External id": 2472787,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035424.398, "dur": 0.596,
    "args": {
      "External id": 2472788,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035425.295, "dur": 1.347,
    "args": {
      "External id": 2472789,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035425.961, "dur": 0.616,
    "args": {
      "External id": 2472790,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035426.987, "dur": 2.822,
    "args": {
      "External id": 2472791,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035429.238, "dur": 0.500,
    "args": {
      "External id": 2472792,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035430.045, "dur": 1.374,
    "args": {
      "External id": 2472793,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035430.760, "dur": 0.587,
    "args": {
      "External id": 2472794,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035431.636, "dur": 2.438,
    "args": {
      "External id": 2472795,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035432.514, "dur": 1.494,
    "args": {
      "External id": 2472796,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035434.320, "dur": 1.208,
    "args": {
      "External id": 2472797,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035434.967, "dur": 0.494,
    "args": {
      "External id": 2472798,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035435.748, "dur": 1.782,
    "args": {
      "External id": 2472799,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035436.791, "dur": 0.510,
    "args": {
      "External id": 2472800,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035439.043, "dur": 2.662,
    "args": {
      "External id": 2472801,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035441.217, "dur": 0.422,
    "args": {
      "External id": 2472802,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035441.947, "dur": 1.363,
    "args": {
      "External id": 2472803,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035442.847, "dur": 0.400,
    "args": {
      "External id": 2472804,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035443.549, "dur": 1.207,
    "args": {
      "External id": 2472805,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035444.120, "dur": 0.571,
    "args": {
      "External id": 2472806,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035444.999, "dur": 2.718,
    "args": {
      "External id": 2472807,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035447.219, "dur": 0.427,
    "args": {
      "External id": 2472808,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035447.987, "dur": 1.172,
    "args": {
      "External id": 2472809,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035448.646, "dur": 0.443,
    "args": {
      "External id": 2472810,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035449.378, "dur": 2.676,
    "args": {
      "External id": 2472811,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035450.367, "dur": 1.522,
    "args": {
      "External id": 2472812,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035452.295, "dur": 1.223,
    "args": {
      "External id": 2472813,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035452.912, "dur": 0.540,
    "args": {
      "External id": 2472814,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035453.739, "dur": 1.781,
    "args": {
      "External id": 2472815,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035454.585, "dur": 0.870,
    "args": {
      "External id": 2472816,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035457.266, "dur": 2.703,
    "args": {
      "External id": 2472817,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035459.344, "dur": 0.553,
    "args": {
      "External id": 2472818,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035460.245, "dur": 1.953,
    "args": {
      "External id": 2472819,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035461.577, "dur": 0.554,
    "args": {
      "External id": 2472820,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035462.455, "dur": 1.457,
    "args": {
      "External id": 2472821,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035463.195, "dur": 0.654,
    "args": {
      "External id": 2472822,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035464.232, "dur": 2.813,
    "args": {
      "External id": 2472823,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035466.569, "dur": 0.408,
    "args": {
      "External id": 2472824,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035467.298, "dur": 1.301,
    "args": {
      "External id": 2472825,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035468.022, "dur": 0.409,
    "args": {
      "External id": 2472826,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035468.816, "dur": 2.788,
    "args": {
      "External id": 2472827,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035469.663, "dur": 1.777,
    "args": {
      "External id": 2472828,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035471.853, "dur": 1.256,
    "args": {
      "External id": 2472829,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035472.449, "dur": 0.590,
    "args": {
      "External id": 2472830,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035494.691, "dur": 146.397,
    "args": {
      "External id": 2472831,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035744.120, "dur": 125.516,
    "args": {
      "External id": 2472832,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035803.650, "dur": 46.150,
    "args": {
      "External id": 2472833,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336756, "tid": 1336756,
    "ts": 1514658035816.990, "dur": 1.186,
    "args": {
      "External id": 2472834,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Redistribute", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036295.251, "dur": 1120.188,
    "args": {
      "External id": 2472835,"Sequence number": 24740656, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "False"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036348.065, "dur": 59.935,
    "args": {
      "External id": 2472836,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036353.483, "dur": 1.642,
    "args": {
      "External id": 2472837,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036356.996, "dur": 0.725,
    "args": {
      "External id": 2472838,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::all_reduce", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036433.751, "dur": 501.810,
    "args": {
      "External id": 2472839,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["float", "", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036439.258, "dur": 43.524,
    "args": {
      "External id": 2472840,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036442.844, "dur": 7.971,
    "args": {
      "External id": 2472841,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "0"], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036446.471, "dur": 3.647,
    "args": {
      "External id": 2472842,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036452.084, "dur": 30.209,
    "args": {
      "External id": 2472843,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::allreduce_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036492.488, "dur": 439.340,
    "args": {
      "External id": 2472844,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "-1"], "Input type": ["TensorList", "", "", "", "Scalar"], "Input Strides": [[[]], [], [], [], []], "Input Dims": [[[]], [], [], [], []], "Ev Idx": 11299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036523.605, "dur": 400.546,
    "args": {
      "External id": 2472845,"Record function id": 0, "Collective name": "allreduce", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 3, "Input Strides": [[[]], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "3", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 1, "Process Group Name": "0", "Input type": ["TensorList", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[[]], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 11300, "In msg nelems": 1
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:all_reduce", "pid": 1336756, "tid": 1336756,
    "ts": 1514658036559.262, "dur": 358.392,
    "args": {
      "External id": 2472846,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037140.102, "dur": 234.396,
    "args": {
      "External id": 2472847,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::wait_tensor", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037236.452, "dur": 38.034,
    "args": {
      "External id": 2472848,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037257.216, "dur": 6.181,
    "args": {
      "External id": 2472849,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "3", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 11304, "In msg nelems": 0, "Rank": 3, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037304.435, "dur": 60.837,
    "args": {
      "External id": 2472850,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037307.596, "dur": 1.420,
    "args": {
      "External id": 2472851,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037310.370, "dur": 0.700,
    "args": {
      "External id": 2472852,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_ToTorchTensor", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037431.292, "dur": 20.920,
    "args": {
      "External id": 2472853,"Sequence number": 24740657, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037440.912, "dur": 8.216,
    "args": {
      "External id": 2472854,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037443.188, "dur": 5.753,
    "args": {
      "External id": 2472855,"Record function id": 0, "Concrete Inputs": ["", "[]"], "Input type": ["float", "ScalarList"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037825.181, "dur": 33.748,
    "args": {
      "External id": 2472856,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "double", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reciprocal", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037872.897, "dur": 24.985,
    "args": {
      "External id": 2472857,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mul", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037911.638, "dur": 24.753,
    "args": {
      "External id": 2472858,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "double"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clamp", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037948.513, "dur": 22.936,
    "args": {
      "External id": 2472859,"Record function id": 0, "Concrete Inputs": ["", "", "1."], "Input type": ["float", "", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658037951.304, "dur": 0.672,
    "args": {
      "External id": 2472860,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336756, "tid": 1336756,
    "ts": 1514658038023.099, "dur": 0.529,
    "args": {
      "External id": 2472861,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 11316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658038145.497, "dur": 814.863,
    "args": {
      "External id": 2472862,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658038654.695, "dur": 276.231,
    "args": {
      "External id": 2472863,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isnan", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039035.710, "dur": 33.400,
    "args": {
      "External id": 2472864,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039040.026, "dur": 28.280,
    "args": {
      "External id": 2472865,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039073.728, "dur": 167.683,
    "args": {
      "External id": 2472866,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039075.543, "dur": 165.605,
    "args": {
      "External id": 2472867,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039077.523, "dur": 163.299,
    "args": {
      "External id": 2472868,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isinf", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039246.176, "dur": 59.600,
    "args": {
      "External id": 2472869,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039249.287, "dur": 31.957,
    "args": {
      "External id": 2472870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039256.502, "dur": 2.774,
    "args": {
      "External id": 2472871,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039261.151, "dur": 19.834,
    "args": {
      "External id": 2472872,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], [1]], "Input Dims": [[], [0]], "Ev Idx": 11327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039265.549, "dur": 2.637,
    "args": {
      "External id": 2472873,"Record function id": 0, "Concrete Inputs": ["", "[]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 11328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039284.391, "dur": 20.513,
    "args": {
      "External id": 2472874,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039307.748, "dur": 37.985,
    "args": {
      "External id": 2472875,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039310.299, "dur": 35.265,
    "args": {
      "External id": 2472876,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039310.971, "dur": 34.212,
    "args": {
      "External id": 2472877,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11332
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#OptimizersContainer.step", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039377.829, "dur": 5745.918,
    "args": {
      "External id": 2472878,"Record function id": 0, "Ev Idx": 11333
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#AdamW.step", "pid": 1336756, "tid": 1336756,
    "ts": 1514658039410.540, "dur": 5691.405,
    "args": {
      "External id": 2472879,"Record function id": 0, "Ev Idx": 11334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_add_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040674.695, "dur": 263.825,
    "args": {
      "External id": 2472880,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040693.910, "dur": 1.106,
    "args": {
      "External id": 2472881,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040696.353, "dur": 0.110,
    "args": {
      "External id": 2472882,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040696.923, "dur": 0.241,
    "args": {
      "External id": 2472883,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040697.663, "dur": 0.157,
    "args": {
      "External id": 2472884,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040698.304, "dur": 0.205,
    "args": {
      "External id": 2472885,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040698.960, "dur": 0.209,
    "args": {
      "External id": 2472886,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040699.522, "dur": 0.352,
    "args": {
      "External id": 2472887,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040700.234, "dur": 0.281,
    "args": {
      "External id": 2472888,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040700.973, "dur": 0.081,
    "args": {
      "External id": 2472889,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040701.454, "dur": 0.096,
    "args": {
      "External id": 2472890,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040701.909, "dur": 0.067,
    "args": {
      "External id": 2472891,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040702.355, "dur": 0.194,
    "args": {
      "External id": 2472892,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040702.828, "dur": 0.065,
    "args": {
      "External id": 2472893,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040703.253, "dur": 0.064,
    "args": {
      "External id": 2472894,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040703.652, "dur": 0.064,
    "args": {
      "External id": 2472895,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040704.068, "dur": 0.069,
    "args": {
      "External id": 2472896,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040704.484, "dur": 0.090,
    "args": {
      "External id": 2472897,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040704.942, "dur": 0.276,
    "args": {
      "External id": 2472898,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040705.610, "dur": 0.065,
    "args": {
      "External id": 2472899,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040706.042, "dur": 0.186,
    "args": {
      "External id": 2472900,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040706.573, "dur": 0.173,
    "args": {
      "External id": 2472901,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040707.208, "dur": 0.195,
    "args": {
      "External id": 2472902,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040707.725, "dur": 0.191,
    "args": {
      "External id": 2472903,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040708.233, "dur": 0.195,
    "args": {
      "External id": 2472904,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040708.725, "dur": 0.243,
    "args": {
      "External id": 2472905,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040709.357, "dur": 0.083,
    "args": {
      "External id": 2472906,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040709.780, "dur": 0.067,
    "args": {
      "External id": 2472907,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040710.200, "dur": 0.076,
    "args": {
      "External id": 2472908,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040710.635, "dur": 0.067,
    "args": {
      "External id": 2472909,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040711.068, "dur": 0.067,
    "args": {
      "External id": 2472910,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040711.456, "dur": 0.071,
    "args": {
      "External id": 2472911,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040711.969, "dur": 0.068,
    "args": {
      "External id": 2472912,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040712.381, "dur": 0.068,
    "args": {
      "External id": 2472913,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040712.763, "dur": 0.072,
    "args": {
      "External id": 2472914,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040713.108, "dur": 0.067,
    "args": {
      "External id": 2472915,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040713.452, "dur": 0.068,
    "args": {
      "External id": 2472916,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040713.913, "dur": 0.051,
    "args": {
      "External id": 2472917,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040714.331, "dur": 0.066,
    "args": {
      "External id": 2472918,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040714.777, "dur": 0.067,
    "args": {
      "External id": 2472919,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040715.340, "dur": 0.064,
    "args": {
      "External id": 2472920,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040715.731, "dur": 0.064,
    "args": {
      "External id": 2472921,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040716.135, "dur": 0.066,
    "args": {
      "External id": 2472922,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040716.743, "dur": 0.069,
    "args": {
      "External id": 2472923,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040717.200, "dur": 0.068,
    "args": {
      "External id": 2472924,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040717.582, "dur": 0.066,
    "args": {
      "External id": 2472925,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040717.988, "dur": 0.066,
    "args": {
      "External id": 2472926,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040718.397, "dur": 0.071,
    "args": {
      "External id": 2472927,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040718.871, "dur": 0.050,
    "args": {
      "External id": 2472928,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040719.338, "dur": 0.068,
    "args": {
      "External id": 2472929,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040719.691, "dur": 0.069,
    "args": {
      "External id": 2472930,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040720.180, "dur": 0.065,
    "args": {
      "External id": 2472931,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040720.631, "dur": 0.296,
    "args": {
      "External id": 2472932,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040721.271, "dur": 0.098,
    "args": {
      "External id": 2472933,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040721.831, "dur": 0.178,
    "args": {
      "External id": 2472934,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040722.366, "dur": 0.198,
    "args": {
      "External id": 2472935,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040723.002, "dur": 0.206,
    "args": {
      "External id": 2472936,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040723.521, "dur": 0.195,
    "args": {
      "External id": 2472937,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040724.093, "dur": 0.195,
    "args": {
      "External id": 2472938,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040724.622, "dur": 0.067,
    "args": {
      "External id": 2472939,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040725.064, "dur": 0.264,
    "args": {
      "External id": 2472940,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040725.931, "dur": 0.077,
    "args": {
      "External id": 2472941,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040726.328, "dur": 0.080,
    "args": {
      "External id": 2472942,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040726.847, "dur": 0.227,
    "args": {
      "External id": 2472943,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040727.396, "dur": 0.091,
    "args": {
      "External id": 2472944,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040727.941, "dur": 0.204,
    "args": {
      "External id": 2472945,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040728.527, "dur": 0.088,
    "args": {
      "External id": 2472946,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040729.170, "dur": 0.051,
    "args": {
      "External id": 2472947,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040729.611, "dur": 0.063,
    "args": {
      "External id": 2472948,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040730.066, "dur": 0.065,
    "args": {
      "External id": 2472949,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040730.525, "dur": 0.062,
    "args": {
      "External id": 2472950,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040730.917, "dur": 0.068,
    "args": {
      "External id": 2472951,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040731.491, "dur": 0.063,
    "args": {
      "External id": 2472952,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040731.826, "dur": 0.068,
    "args": {
      "External id": 2472953,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040732.297, "dur": 0.053,
    "args": {
      "External id": 2472954,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040732.730, "dur": 0.065,
    "args": {
      "External id": 2472955,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040733.306, "dur": 0.067,
    "args": {
      "External id": 2472956,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040733.731, "dur": 0.067,
    "args": {
      "External id": 2472957,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040734.096, "dur": 0.064,
    "args": {
      "External id": 2472958,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040734.557, "dur": 0.056,
    "args": {
      "External id": 2472959,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040735.020, "dur": 0.071,
    "args": {
      "External id": 2472960,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040755.135, "dur": 0.104,
    "args": {
      "External id": 2472961,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040755.694, "dur": 0.052,
    "args": {
      "External id": 2472962,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040756.110, "dur": 0.069,
    "args": {
      "External id": 2472963,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040756.552, "dur": 0.070,
    "args": {
      "External id": 2472964,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040756.919, "dur": 0.062,
    "args": {
      "External id": 2472965,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040757.358, "dur": 0.067,
    "args": {
      "External id": 2472966,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040757.744, "dur": 0.064,
    "args": {
      "External id": 2472967,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040758.172, "dur": 0.065,
    "args": {
      "External id": 2472968,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040758.560, "dur": 0.061,
    "args": {
      "External id": 2472969,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040758.952, "dur": 0.059,
    "args": {
      "External id": 2472970,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040759.433, "dur": 0.066,
    "args": {
      "External id": 2472971,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040759.846, "dur": 0.060,
    "args": {
      "External id": 2472972,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040760.204, "dur": 0.058,
    "args": {
      "External id": 2472973,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040760.579, "dur": 0.066,
    "args": {
      "External id": 2472974,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040760.908, "dur": 0.061,
    "args": {
      "External id": 2472975,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040761.332, "dur": 0.065,
    "args": {
      "External id": 2472976,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040761.763, "dur": 0.049,
    "args": {
      "External id": 2472977,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040762.253, "dur": 0.054,
    "args": {
      "External id": 2472978,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040762.637, "dur": 0.066,
    "args": {
      "External id": 2472979,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040762.976, "dur": 0.066,
    "args": {
      "External id": 2472980,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040763.383, "dur": 0.063,
    "args": {
      "External id": 2472981,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040763.728, "dur": 0.071,
    "args": {
      "External id": 2472982,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040764.115, "dur": 0.074,
    "args": {
      "External id": 2472983,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040764.477, "dur": 0.062,
    "args": {
      "External id": 2472984,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040764.900, "dur": 0.047,
    "args": {
      "External id": 2472985,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040765.298, "dur": 0.065,
    "args": {
      "External id": 2472986,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040765.873, "dur": 0.053,
    "args": {
      "External id": 2472987,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040766.319, "dur": 0.065,
    "args": {
      "External id": 2472988,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040766.728, "dur": 0.064,
    "args": {
      "External id": 2472989,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040767.131, "dur": 0.066,
    "args": {
      "External id": 2472990,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040767.519, "dur": 0.063,
    "args": {
      "External id": 2472991,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040767.935, "dur": 0.051,
    "args": {
      "External id": 2472992,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040768.408, "dur": 0.052,
    "args": {
      "External id": 2472993,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040768.750, "dur": 0.064,
    "args": {
      "External id": 2472994,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040769.175, "dur": 0.064,
    "args": {
      "External id": 2472995,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040769.518, "dur": 0.066,
    "args": {
      "External id": 2472996,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040770.021, "dur": 0.067,
    "args": {
      "External id": 2472997,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040770.464, "dur": 0.065,
    "args": {
      "External id": 2472998,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040770.966, "dur": 0.046,
    "args": {
      "External id": 2472999,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040771.329, "dur": 0.068,
    "args": {
      "External id": 2473000,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040771.768, "dur": 0.064,
    "args": {
      "External id": 2473001,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040772.144, "dur": 0.064,
    "args": {
      "External id": 2473002,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040772.677, "dur": 0.064,
    "args": {
      "External id": 2473003,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040773.224, "dur": 0.063,
    "args": {
      "External id": 2473004,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040773.744, "dur": 0.065,
    "args": {
      "External id": 2473005,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040774.134, "dur": 0.066,
    "args": {
      "External id": 2473006,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040774.532, "dur": 0.067,
    "args": {
      "External id": 2473007,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040774.928, "dur": 0.064,
    "args": {
      "External id": 2473008,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040775.380, "dur": 0.068,
    "args": {
      "External id": 2473009,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040775.806, "dur": 0.064,
    "args": {
      "External id": 2473010,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040781.804, "dur": 0.078,
    "args": {
      "External id": 2473011,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040783.304, "dur": 0.064,
    "args": {
      "External id": 2473012,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040783.704, "dur": 0.065,
    "args": {
      "External id": 2473013,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040784.172, "dur": 0.069,
    "args": {
      "External id": 2473014,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040784.592, "dur": 0.065,
    "args": {
      "External id": 2473015,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040784.912, "dur": 0.067,
    "args": {
      "External id": 2473016,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040785.436, "dur": 0.069,
    "args": {
      "External id": 2473017,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040785.864, "dur": 0.046,
    "args": {
      "External id": 2473018,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040786.279, "dur": 0.066,
    "args": {
      "External id": 2473019,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040786.700, "dur": 0.064,
    "args": {
      "External id": 2473020,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040787.144, "dur": 0.068,
    "args": {
      "External id": 2473021,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040787.822, "dur": 0.061,
    "args": {
      "External id": 2473022,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040788.187, "dur": 0.070,
    "args": {
      "External id": 2473023,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040788.590, "dur": 0.062,
    "args": {
      "External id": 2473024,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040788.958, "dur": 0.067,
    "args": {
      "External id": 2473025,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040789.381, "dur": 0.056,
    "args": {
      "External id": 2473026,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040789.764, "dur": 0.065,
    "args": {
      "External id": 2473027,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040790.239, "dur": 0.053,
    "args": {
      "External id": 2473028,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040790.620, "dur": 0.066,
    "args": {
      "External id": 2473029,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040791.037, "dur": 0.069,
    "args": {
      "External id": 2473030,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040791.404, "dur": 0.068,
    "args": {
      "External id": 2473031,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040791.798, "dur": 0.066,
    "args": {
      "External id": 2473032,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040792.151, "dur": 0.064,
    "args": {
      "External id": 2473033,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040792.766, "dur": 0.067,
    "args": {
      "External id": 2473034,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040793.212, "dur": 0.054,
    "args": {
      "External id": 2473035,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040793.651, "dur": 0.064,
    "args": {
      "External id": 2473036,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040794.030, "dur": 0.073,
    "args": {
      "External id": 2473037,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040794.471, "dur": 0.068,
    "args": {
      "External id": 2473038,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040794.884, "dur": 0.074,
    "args": {
      "External id": 2473039,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040795.332, "dur": 0.055,
    "args": {
      "External id": 2473040,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040795.712, "dur": 0.067,
    "args": {
      "External id": 2473041,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040796.106, "dur": 0.065,
    "args": {
      "External id": 2473042,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040796.451, "dur": 0.068,
    "args": {
      "External id": 2473043,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040796.880, "dur": 0.069,
    "args": {
      "External id": 2473044,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040797.246, "dur": 0.057,
    "args": {
      "External id": 2473045,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040797.594, "dur": 0.070,
    "args": {
      "External id": 2473046,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040797.977, "dur": 0.070,
    "args": {
      "External id": 2473047,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040798.375, "dur": 0.070,
    "args": {
      "External id": 2473048,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040798.766, "dur": 0.066,
    "args": {
      "External id": 2473049,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040799.211, "dur": 0.056,
    "args": {
      "External id": 2473050,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040799.639, "dur": 0.065,
    "args": {
      "External id": 2473051,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040800.284, "dur": 0.068,
    "args": {
      "External id": 2473052,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040800.784, "dur": 0.055,
    "args": {
      "External id": 2473053,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040801.104, "dur": 0.070,
    "args": {
      "External id": 2473054,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040801.490, "dur": 0.070,
    "args": {
      "External id": 2473055,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040802.075, "dur": 0.069,
    "args": {
      "External id": 2473056,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040802.519, "dur": 0.065,
    "args": {
      "External id": 2473057,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040803.005, "dur": 0.056,
    "args": {
      "External id": 2473058,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040803.378, "dur": 0.067,
    "args": {
      "External id": 2473059,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040803.847, "dur": 0.068,
    "args": {
      "External id": 2473060,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040804.324, "dur": 0.071,
    "args": {
      "External id": 2473061,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040804.679, "dur": 0.067,
    "args": {
      "External id": 2473062,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040805.025, "dur": 0.072,
    "args": {
      "External id": 2473063,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040805.483, "dur": 0.070,
    "args": {
      "External id": 2473064,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040805.899, "dur": 0.053,
    "args": {
      "External id": 2473065,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040806.298, "dur": 0.065,
    "args": {
      "External id": 2473066,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040806.684, "dur": 0.066,
    "args": {
      "External id": 2473067,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040807.114, "dur": 0.066,
    "args": {
      "External id": 2473068,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040807.505, "dur": 0.068,
    "args": {
      "External id": 2473069,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040807.879, "dur": 0.068,
    "args": {
      "External id": 2473070,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040808.299, "dur": 0.065,
    "args": {
      "External id": 2473071,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040808.794, "dur": 0.068,
    "args": {
      "External id": 2473072,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040809.238, "dur": 0.067,
    "args": {
      "External id": 2473073,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040809.644, "dur": 0.067,
    "args": {
      "External id": 2473074,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040810.062, "dur": 0.066,
    "args": {
      "External id": 2473075,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040810.512, "dur": 0.066,
    "args": {
      "External id": 2473076,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040811.044, "dur": 0.073,
    "args": {
      "External id": 2473077,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040811.586, "dur": 0.071,
    "args": {
      "External id": 2473078,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040811.938, "dur": 0.070,
    "args": {
      "External id": 2473079,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040812.403, "dur": 0.066,
    "args": {
      "External id": 2473080,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040812.843, "dur": 0.063,
    "args": {
      "External id": 2473081,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040813.186, "dur": 0.061,
    "args": {
      "External id": 2473082,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040813.617, "dur": 0.066,
    "args": {
      "External id": 2473083,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040813.970, "dur": 0.064,
    "args": {
      "External id": 2473084,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040814.293, "dur": 0.067,
    "args": {
      "External id": 2473085,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040814.710, "dur": 0.070,
    "args": {
      "External id": 2473086,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040815.237, "dur": 0.068,
    "args": {
      "External id": 2473087,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040815.760, "dur": 0.068,
    "args": {
      "External id": 2473088,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040816.183, "dur": 0.063,
    "args": {
      "External id": 2473089,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040816.546, "dur": 0.066,
    "args": {
      "External id": 2473090,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040816.939, "dur": 0.068,
    "args": {
      "External id": 2473091,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040817.264, "dur": 0.065,
    "args": {
      "External id": 2473092,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040817.650, "dur": 0.069,
    "args": {
      "External id": 2473093,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040818.101, "dur": 0.067,
    "args": {
      "External id": 2473094,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040818.529, "dur": 0.065,
    "args": {
      "External id": 2473095,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040818.958, "dur": 0.051,
    "args": {
      "External id": 2473096,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040819.455, "dur": 0.066,
    "args": {
      "External id": 2473097,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040819.791, "dur": 0.065,
    "args": {
      "External id": 2473098,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040820.161, "dur": 0.066,
    "args": {
      "External id": 2473099,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040820.559, "dur": 0.063,
    "args": {
      "External id": 2473100,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040821.128, "dur": 0.068,
    "args": {
      "External id": 2473101,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040821.461, "dur": 0.068,
    "args": {
      "External id": 2473102,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040821.908, "dur": 0.063,
    "args": {
      "External id": 2473103,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040822.322, "dur": 0.066,
    "args": {
      "External id": 2473104,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040822.835, "dur": 0.053,
    "args": {
      "External id": 2473105,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040823.187, "dur": 0.065,
    "args": {
      "External id": 2473106,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040823.615, "dur": 0.061,
    "args": {
      "External id": 2473107,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040824.108, "dur": 0.049,
    "args": {
      "External id": 2473108,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040824.532, "dur": 0.067,
    "args": {
      "External id": 2473109,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040824.970, "dur": 0.071,
    "args": {
      "External id": 2473110,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040825.329, "dur": 0.059,
    "args": {
      "External id": 2473111,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040825.636, "dur": 0.057,
    "args": {
      "External id": 2473112,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040826.011, "dur": 0.066,
    "args": {
      "External id": 2473113,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040826.388, "dur": 0.066,
    "args": {
      "External id": 2473114,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040826.812, "dur": 0.057,
    "args": {
      "External id": 2473115,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040827.163, "dur": 0.064,
    "args": {
      "External id": 2473116,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040827.543, "dur": 0.065,
    "args": {
      "External id": 2473117,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040828.119, "dur": 0.066,
    "args": {
      "External id": 2473118,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040828.509, "dur": 0.080,
    "args": {
      "External id": 2473119,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040829.134, "dur": 0.070,
    "args": {
      "External id": 2473120,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040829.526, "dur": 0.056,
    "args": {
      "External id": 2473121,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040829.940, "dur": 0.065,
    "args": {
      "External id": 2473122,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040830.348, "dur": 0.067,
    "args": {
      "External id": 2473123,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040830.832, "dur": 0.064,
    "args": {
      "External id": 2473124,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040831.220, "dur": 0.068,
    "args": {
      "External id": 2473125,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040831.745, "dur": 0.050,
    "args": {
      "External id": 2473126,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040832.163, "dur": 0.071,
    "args": {
      "External id": 2473127,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040832.598, "dur": 0.070,
    "args": {
      "External id": 2473128,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040833.020, "dur": 0.063,
    "args": {
      "External id": 2473129,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040833.395, "dur": 0.064,
    "args": {
      "External id": 2473130,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040833.806, "dur": 0.069,
    "args": {
      "External id": 2473131,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040834.210, "dur": 0.069,
    "args": {
      "External id": 2473132,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040834.651, "dur": 0.068,
    "args": {
      "External id": 2473133,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040835.152, "dur": 0.070,
    "args": {
      "External id": 2473134,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040835.587, "dur": 0.054,
    "args": {
      "External id": 2473135,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040836.012, "dur": 0.068,
    "args": {
      "External id": 2473136,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040836.430, "dur": 0.059,
    "args": {
      "External id": 2473137,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040836.810, "dur": 0.063,
    "args": {
      "External id": 2473138,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040837.311, "dur": 0.066,
    "args": {
      "External id": 2473139,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040837.710, "dur": 0.049,
    "args": {
      "External id": 2473140,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040838.075, "dur": 0.066,
    "args": {
      "External id": 2473141,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040838.388, "dur": 0.068,
    "args": {
      "External id": 2473142,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040838.963, "dur": 0.052,
    "args": {
      "External id": 2473143,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040839.393, "dur": 0.062,
    "args": {
      "External id": 2473144,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040839.831, "dur": 0.061,
    "args": {
      "External id": 2473145,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040840.495, "dur": 0.066,
    "args": {
      "External id": 2473146,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040840.862, "dur": 0.053,
    "args": {
      "External id": 2473147,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040841.359, "dur": 0.066,
    "args": {
      "External id": 2473148,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040841.665, "dur": 0.057,
    "args": {
      "External id": 2473149,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040842.318, "dur": 0.064,
    "args": {
      "External id": 2473150,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040842.628, "dur": 0.050,
    "args": {
      "External id": 2473151,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040843.292, "dur": 0.063,
    "args": {
      "External id": 2473152,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040843.595, "dur": 0.058,
    "args": {
      "External id": 2473153,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040844.234, "dur": 0.065,
    "args": {
      "External id": 2473154,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040844.568, "dur": 0.050,
    "args": {
      "External id": 2473155,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040845.060, "dur": 0.066,
    "args": {
      "External id": 2473156,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040845.367, "dur": 0.054,
    "args": {
      "External id": 2473157,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040845.908, "dur": 0.068,
    "args": {
      "External id": 2473158,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040846.246, "dur": 0.053,
    "args": {
      "External id": 2473159,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040846.873, "dur": 0.065,
    "args": {
      "External id": 2473160,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040847.176, "dur": 0.056,
    "args": {
      "External id": 2473161,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040847.711, "dur": 0.069,
    "args": {
      "External id": 2473162,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040848.027, "dur": 0.051,
    "args": {
      "External id": 2473163,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040848.535, "dur": 0.065,
    "args": {
      "External id": 2473164,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040848.848, "dur": 0.053,
    "args": {
      "External id": 2473165,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040849.484, "dur": 0.071,
    "args": {
      "External id": 2473166,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040849.796, "dur": 0.057,
    "args": {
      "External id": 2473167,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040850.321, "dur": 0.065,
    "args": {
      "External id": 2473168,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040850.627, "dur": 0.054,
    "args": {
      "External id": 2473169,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040851.156, "dur": 0.073,
    "args": {
      "External id": 2473170,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336756, "tid": 1336756,
    "ts": 1514658040851.469, "dur": 0.051,
    "args": {
      "External id": 2473171,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658041463.951, "dur": 3511.216,
    "args": {
      "External id": 2473172,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336756, "tid": 1336756,
    "ts": 1514658044470.463, "dur": 348.010,
    "args": {
      "External id": 2473173,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11628
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 0,
    "args": {
      "name": "python3.12"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 0,
    "args": {
      "labels": "CPU"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 0,
    "args": {
      "sort_index": 1336756
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 1381189,
    "args": {
      "name": "thread 1381189 (pt_autograd_3)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 1381189,
    "args": {
      "sort_index": 1381189
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 1381189,
    "args": {
      "name": "thread 1381189 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 1381189,
    "args": {
      "sort_index": 1381189
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 1336756,
    "args": {
      "name": "thread 1336756 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655937170.225, "pid": 1336756, "tid": 1336756,
    "args": {
      "sort_index": 1336756
    }
  },
  {
    "ph": "X", "cat": "Trace", "ts": 1514655937102.617, "dur": 2109633.691,
    "pid": "Spans", "tid": "PyTorch Profiler",
    "name": "PyTorch Profiler (0)",
    "args": {
      "Op count": 0
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655937102.617,
    "pid": "Spans", "tid": 0,
    "args": {
      "sort_index": 536870912
    }
  },
  {
    "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g",
    "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1514655937102.617
  },
  {
    "name": "Record Window End", "ph": "i", "s": "g",
    "pid": "", "tid": "", "ts": 1514658109347.964
  }
  ],
  "traceName": "exp/mtp.1B.batch16.seqlen4096.context4096.warmup2000.update1.steps200000.lr2e-4.cosine/profile_trace/iteration_102912/rank3_trace.json",
  "displayTimeUnit": "ms",
  "baseTimeNanoseconds": 1751410836000000000
}