when you mean “without using any libraries” you are referring to the low level API (no Keras)? if its the case you can always build stuff using common python and constructing models with tf.Module  |  TensorFlow Core v2.5.0 1
for example, this is the code to implement a simple conv layer without Keras:
weights initialization:
def weights(name, shape, mean=0.0, stddev=0.02):
    var = tf.Variable(tf.random_normal_initializer(mean=mean, stddev=stddev)(shape), name=name)
    return var

layer:
class Conv2D(tf.Module):
    def __init__(self,  out_feats=8 ks=3, use_relu=False, name=None):
        super(Conv2D, self).__init__(name=name)
        self.use_relu = use_relu
        self.out_feats = out_feats
        self.ks = ks

    @tf.Module.with_name_scope
    def __call__(self, input):
        if not hasattr(self, 'weights'):
            self.weights = weights('weights', (self.ks,  self.ks, input.get_shape()[-1],  self.out_feats))
        conv = tf.nn.conv2d(input, self.weights, strides=[1, 1, 1, 1], padding='SAME')

        if self.use_relu:
            output = tf.nn.relu(conv)
        else:
            output = conv
        return output