text: Title: TypeError: unary_unary() got an unexpected keyword argument '_registered_method'
  Body: ### What happened?
  
  When I run the following scripts:
  
  ```
  import kubeflow.katib as katib
  
  def train_mnist_model(parameters):
      import tensorflow as tf
      import kubeflow.katib as katib
      import numpy as np
      import logging
  
      logging.basicConfig(
          format="%(asctime)s %(levelname)-8s %(message)s",
          datefmt="%Y-%m-%dT%H:%M:%SZ",
          level=logging.INFO,
      )
      logging.info("--------------------------------------------------------------------------------------")
      logging.info(f"Input Parameters: {parameters}")
      logging.info("--------------------------------------------------------------------------------------\n\n")
  
  
      # Get HyperParameters from the input params dict.
      lr = float(parameters["lr"])
      num_epoch = int(parameters["num_epoch"])
  
      # Set dist parameters and strategy.
      is_dist = parameters["is_dist"]
      num_workers = parameters["num_workers"]
      batch_size_per_worker = 64
      batch_size_global = batch_size_per_worker * num_workers
      strategy = tf.distribute.MultiWorkerMirroredStrategy(
          communication_options=tf.distribute.experimental.CommunicationOptions(
              implementation=tf.distribute.experimental.CollectiveCommunication.RING
          )
      )
  
      # Callback class for logging training.
      # Katib parses metrics in this format: <metric-name>=<metric-value>.
      class CustomCallback(tf.keras.callbacks.Callback):
          def on_epoch_end(self, epoch, logs=None):
              katib.report_metrics({
                  "accuracy": logs["accuracy"],
                  "logs": logs["loss"],
              })
              
  
      # Prepare MNIST Dataset.
      def mnist_dataset(batch_size):
          (x_train, y_train), _ = tf.keras.datasets.mnist.load_data()
          x_train = x_train / np.float32(255)
          y_train = y_train.astype(np.int64)
          train_dataset = (
              tf.data.Dataset.from_tensor_slices((x_train, y_train))
              .shuffle(60000)
              .repeat()
              .batch(batch_size)
          )
          return train_dataset
  
      # Build and compile CNN Model.
      def build_and_compile_cnn_model():
          model = tf.keras.Sequential(
              [
                  tf.keras.layers.InputLayer(input_shape=(28, 28)),
                  tf.keras.layers.Reshape(target_shape=(28, 28, 1)),
                  tf.keras.layers.Conv2D(32, 3, activation="relu"),
                  tf.keras.layers.Flatten(),
                  tf.keras.layers.Dense(128, activation="relu"),
                  tf.keras.layers.Dense(10),
              ]
          )
          model.compile(
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              optimizer=tf.keras.optimizers.SGD(learning_rate=lr),
              metrics=["accuracy"],
          )
          return model
      
      # Download Dataset.
      dataset = mnist_dataset(batch_size_global)
  
      # For dist strategy we should build model under scope().
      if is_dist:
          logging.info("Running Distributed Training")
          logging.info("--------------------------------------------------------------------------------------\n\n")
          with strategy.scope():
              model = build_and_compile_cnn_model()
      else:
          logging.info("Running Single Worker Training")
          logging.info("--------------------------------------------------------------------------------------\n\n")
          model = build_and_compile_cnn_model()
      
      # Start Training.
      model.fit(
          dataset,
          epochs=num_epoch,
          steps_per_epoch=70,
          callbacks=[CustomCallback()],
          verbose=0,
      )
  
  # Set parameters with their distribution for HyperParameter Tuning with Katib.
  parameters = {
      "lr": katib.search.double(min=0.1, max=0.2),
      "num_epoch": katib.search.int(min=10, max=15),
      "is_dist": False,
      "num_workers": 1
  }
  
  # Start the Katib Experiment.
  katib_client = katib.KatibClient(namespace="kubeflow")
  katib_client.tune(
      name="tune-mnist",
      objective=train_mnist_model, # Objective function.
      base_image="electronicwaste/tensorflow:git", # tensorflow/tensorflow:2.13.0 + git
      parameters=parameters, # HyperParameters to tune.
      algorithm_name="cmaes", # Alorithm to use.
      objective_metric_name="accuracy", # Katib is going to optimize "accuracy".
      additional_metric_names=["loss"], # Katib is going to collect these metrics in addition to the objective metric.
      max_trial_count=12, # Trial Threshold.
      parallel_trial_count=2,
      packages_to_install=["git+https://github.com/kubeflow/katib.git@master#subdirectory=sdk/python/v1beta1"],
      metrics_collector_config={"kind": "Push"},
  )
  ```
  
  The error happened:
  
  ```
  Traceback (most recent call last):
    File "/tmp/tmp.fGitfCta5x/ephemeral_objective.py", line 97, in <module>
      train_mnist_model({'lr': '0.16377224201308005', 'num_epoch': '13', 'is_dist': False, 'num_workers': 1})
    File "/tmp/tmp.fGitfCta5x/ephemeral_objective.py", line 89, in train_mnist_model
      model.fit(
    File "/usr/local/lib/python3.8/dist-packages/keras/src/utils/traceback_utils.py", line 70, in error_handler
      raise e.with_traceback(filtered_tb) from None
    File "/tmp/tmp.fGitfCta5x/ephemeral_objective.py", line 36, in on_epoch_end
      katib.report_metrics({
    File "/usr/local/lib/python3.8/dist-packages/kubeflow/katib/api/report_metrics.py", line 61, in report_metrics
      client = katib_api_pb2_grpc.DBManagerStub(channel)
    File "/usr/local/lib/python3.8/dist-packages/kubeflow/katib/katib_api_pb2_grpc.py", line 19, in __init__
      self.ReportObservationLog = channel.unary_unary(
  TypeError: unary_unary() got an unexpected keyword argument '_registered_method'
  ```
  
  ### What did you expect to happen?
  
  Run without error.
  
  ### Environment
  
  Kubernetes version:
  ```bash
  $ kubectl version
  Client Version: v1.30.2
  Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
  Server Version: v1.30.1
  ```
  Katib controller version:
  ```bash
  $ kubectl get pods -n kubeflow -l katib.kubeflow.org/component=controller -o jsonpath="{.items[*].spec.containers[*].image}"
  docker.io/kubeflowkatib/katib-controller:lates
  ```
  Katib Python SDK version:
  ```bash
  $ pip show kubeflow-katib
  Name: kubeflow-katib
  Version: 0.17.0
  Summary: Katib Python SDK for APIVersion v1beta1
  Home-page: https://github.com/kubeflow/katib/tree/master/sdk/python/v1beta1
  Author: Kubeflow Authors
  Author-email: premnath.vel@gmail.com
  License: Apache License Version 2.0
  Location: /home/ws/miniconda3/envs/katib/lib/python3.10/site-packages
  Requires: certifi, grpcio, kubernetes, protobuf, setuptools, six, urllib3
  Required-by: 
  ```
  
  Python Packages Version in the Training Container:
  ```bash
  $ pip list
  Package                      Version
  ---------------------------- --------------------
  absl-py                      1.4.0
  astunparse                   1.6.3
  cachetools                   5.3.1
  certifi                      2019.11.28
  chardet                      3.0.4
  dbus-python                  1.2.16
  flatbuffers                  23.5.26
  gast                         0.4.0
  google-auth                  2.21.0
  google-auth-oauthlib         1.0.0
  google-pasta                 0.2.0
  grpcio                       1.56.0
  h5py                         3.9.0
  idna                         2.8
  importlib-metadata           6.7.0
  keras                        2.13.1
  kubeflow-katib               0.17.0
  kubernetes                   30.1.0
  libclang                     16.0.0
  Markdown                     3.4.3
  MarkupSafe                   2.1.3
  numpy                        1.24.3
  oauthlib                     3.2.2
  opt-einsum                   3.3.0
  packaging                    23.1
  pip                          23.1.2
  protobuf                     4.23.3
  pyasn1                       0.5.0
  pyasn1-modules               0.3.0
  PyGObject                    3.36.0
  python-apt                   2.0.1+ubuntu0.20.4.1
  python-dateutil              2.9.0.post0
  PyYAML                       6.0.2
  requests                     2.22.0
  requests-oauthlib            1.3.1
  requests-unixsocket          0.2.0
  rsa                          4.9
  setuptools                   68.0.0
  six                          1.14.0
  tensorboard                  2.13.0
  tensorboard-data-server      0.7.1
  tensorflow-cpu               2.13.0
  tensorflow-estimator         2.13.0
  tensorflow-io-gcs-filesystem 0.32.0
  termcolor                    2.3.0
  typing_extensions            4.5.0
  urllib3                      1.25.8
  websocket-client             1.8.0
  Werkzeug                     2.3.6
  wheel                        0.40.0
  wrapt                        1.15.0
  zipp                         3.15.0
  ```
  
  ### Impacted by this bug?
  
  Give it a 👍 We prioritize the issues with most 👍