Tutorial 8 - TensorFlow

This tutorial is adapted from a 2022 blog post on the website Made With ML, by Goku Mohandas.

We will import TensorFlow and numpy and set the seed for their random number generators for reproducibility.

import numpy as np
import tensorflow as tf

tf_version = tf.__version__

print(f'TF version: {tf_version}')

TF version: 2.8.2

SEED = 1

# Set seed for reproducability
np.random.seed(seed=SEED)
tf.random.set_seed(seed=SEED)

Basics

First, we will cover some basics such as creating TensorFlow tensors and converting from common data structures to TensorFlow tensors.

x = tf.random.normal((2, 3))
print(f'Type: {x.dtype}')
print(f'shape: {x.shape}')
print(f'values:\n{x}')

Type: <dtype: 'float32'>
shape: (2, 3)
values:
[[-1.1012203   1.5457517   0.383644  ]
 [-0.87965786 -1.2246722  -0.9811211 ]]

# Zeros and ones tensors
x = tf.zeros((2, 3), dtype=tf.float64)
print(x)
x = tf.ones((2, 3))
print(x)

# List => Tensor
x = tf.constant([[1, 2, 3],
                  [4, 5, 6]])

print(f'shape: {x.shape}')
print(f'values:\n{x}')

# Numpy array => Tensor
x = tf.constant(np.random.rand(2, 3))
print(f'shape: {x.shape}')
print(f'values:\n{x}')

shape: (2, 3)
values:
[[4.17022005e-01 7.20324493e-01 1.14374817e-04]
 [3.02332573e-01 1.46755891e-01 9.23385948e-02]]

Operations

# Addition
x = tf.random.normal((2, 3))
y = tf.random.normal((2, 3))
z = x + y
print(f'shape: {z.shape}')
print(f'values:\n{z}')

shape: (2, 3)
values:
[[-0.05392435 -1.4948881   0.6654824 ]
 [ 0.4435789   1.0243716   0.5050061 ]]

# Dot product
x = tf.random.normal((2, 3))
y = tf.random.normal((3, 2))
z = tf.matmul(x, y)
print(f'shape: {z.shape}')
print(f'values:\n{z}')

shape: (2, 2)
values:
[[ 2.04325     1.1587756 ]
 [-1.8957058  -0.67201185]]

# Transpose
x = tf.random.normal((2, 3))
print(f"shape: {x.shape}")
print(f"values: \n{x}")
y = tf.transpose(x)
print(f"shape: {y.shape}")
print(f"values: \n{y}")

shape: (2, 3)
values:
[[-1.1771783  -0.90325946  0.8419609 ]
 [-0.06870949 -0.96161884 -0.51533026]]
shape: (3, 2)
values:
[[-1.1771783  -0.06870949]
 [-0.90325946 -0.96161884]
 [ 0.8419609  -0.51533026]]

# Reshape
x = tf.random.normal((2, 3))
z = tf.reshape(x, (3, 2))
print(f"shape: {z.shape}")
print(f"values: \n{z}")

shape: (3, 2)
values:
[[ 0.73264295  0.99835527]
 [ 1.555784    1.0374023 ]
 [-0.0362004  -0.18817899]]

# Dangers of reshaping (unintended consequences)
x = tf.constant([
    [[1,1,1,1], [2,2,2,2], [3,3,3,3]],
    [[10,10,10,10], [20,20,20,20], [30,30,30,30]]
])
print(f"shape: {x.shape}")
print(f"x: \n{x}\n")

a = tf.reshape(x, (x.shape[1], -1))
print(f"\nshape: {a.shape}")
print(f"a: \n{a}\n")

b = tf.transpose(x, perm=[1, 0, 2])
print(f"\nshape: {b.shape}")
print(f"b: \n{b}\n")

c = tf.reshape(b, (b.shape[0], -1))
print(f"\nshape: {c.shape}")
print(f"c: \n{c}")

shape: (2, 3, 4)
x:
[[[ 1  1  1  1]
  [ 2  2  2  2]
  [ 3  3  3  3]]

 [[10 10 10 10]
  [20 20 20 20]
  [30 30 30 30]]]


shape: (3, 8)
a:
[[ 1  1  1  1  2  2  2  2]
 [ 3  3  3  3 10 10 10 10]
 [20 20 20 20 30 30 30 30]]


shape: (3, 2, 4)
b:
[[[ 1  1  1  1]
  [10 10 10 10]]

 [[ 2  2  2  2]
  [20 20 20 20]]

 [[ 3  3  3  3]
  [30 30 30 30]]]


shape: (3, 8)
c:
[[ 1  1  1  1 10 10 10 10]
 [ 2  2  2  2 20 20 20 20]
 [ 3  3  3  3 30 30 30 30]]

# Dimensional operations
x = tf.random.normal((2, 3))
print(f"values: \n{x}")
y = tf.reduce_sum(x, axis=0) # sum over columns
print(f"values: \n{y}")
z = tf.reduce_sum(x, axis=1) # sum over rows
print(f"values: \n{z}")

values:
[[ 0.9868413   0.57056284  0.17946035]
 [ 0.83900064  1.0045967  -0.0642297 ]]
values:
[1.8258419  1.5751595  0.11523065]
values:
[1.7368646 1.7793677]

Indexing

Now we will look at how to extract, separate, and join values from tensors.

x = tf.random.normal((3, 4))
print (f"x: \n{x}")
print()
print(f"x[:1]: \n{x[0]}")
print()
print(f"x[:1, 1:3]: \n{x[:1, 1:3]}")

x:
[[-0.8340743  -1.0207754  -0.6342568   0.51541275]
 [-0.590669    1.6425287   0.60286343 -0.61301523]
 [ 0.6574386   0.16823037  1.4946445   0.8306155 ]]

x[:1]:
[-0.8340743  -1.0207754  -0.6342568   0.51541275]

x[:1, 1:3]:
[[-1.0207754 -0.6342568]]

Slicing

# Select with dimensional indices
x = tf.random.normal((2, 3))
print(f"values: \n{x}")

col_indices = tf.constant([0, 2])
chosen = tf.gather(x, axis=1, indices=col_indices) # values from column 0 & 2
print(f"values: \n{chosen}")

row_indices = tf.constant([0, 1])
col_indices = tf.constant([0, 2])
chosen = tf.gather_nd(x, indices=[row_indices, col_indices]) # values from (0, 0) & (1, 2)
print(f"values: \n{chosen}")

values:
[[-0.92759573  0.06353194  0.26116046]
 [-1.4687079  -0.90832067 -0.87465733]]
values:
[[-0.92759573  0.26116046]
 [-1.4687079  -0.87465733]]
values:
[0.06353194 0.26116046]

Joining

# Concatenation
x = tf.random.normal((2, 3))
print(f"Values: \n{x}")
y = tf.concat([x, x], axis=0) # stack by rows (dim=1 to stack by columns)
print(f"Values: \n{y}")

Values:
[[ 2.260564   -0.6558232  -0.46591297]
 [ 0.4690215   1.026158   -0.11631647]]
Values:
[[ 2.260564   -0.6558232  -0.46591297]
 [ 0.4690215   1.026158   -0.11631647]
 [ 2.260564   -0.6558232  -0.46591297]
 [ 0.4690215   1.026158   -0.11631647]]

Gradients

• \(y = 3x + 2\)

• \(z = \sum{y}/N\)

• \(\frac{\partial(z)}{\partial(x)} = \frac{\partial(z)}{\partial(y)} \frac{\partial(y)}{\partial(x)} = \frac{1}{N} * 3 = \frac{1}{12} * 3 = 0.25\)

# Tensors with gradient book keeping
x = tf.random.normal((3, 4))

# Tensorflow needs graph context to track gradients
with tf.GradientTape() as g:
    g.watch(x)
    y = 3*x + 2
    z = tf.reduce_mean(y)

dz_dx = g.gradient(z, x)

print(dz_dx)

tf.Tensor(
[[0.25 0.25 0.25 0.25]
 [0.25 0.25 0.25 0.25]
 [0.25 0.25 0.25 0.25]], shape=(3, 4), dtype=float32)

CUDA

This section details how to check if we are able to use GPU to accelerate our machine learning or deep learning models.

The Compute Unified Device Architecture or CUDA is a parallel computing platform and API that allows software to use certain types of GPUs for general purpose processing. It is an extension of the C and C++ programming languages.

TensorFlow makes using the GPU quite transparent. If the GPU compatible version of TF is installed along with the proper drivers, TF will use the GPU.

Although training is usually faster on the GPU, depending on model size and hardware specs, it can take quite a while to copy the model and your data to the GPU.

Link to metapackage for easily installing TensorFlow GPU using a conda ‘metapackage’. This is just a special package which installs the required GPU drivers alongside TensorFlow.

# Is CUDA available?
print(tf.test.is_built_with_cuda())

False

# Set device to first gpu (if available)
device = "/gpu:0" if tf.test.is_built_with_cuda() else "cpu"
print(device)

cpu

with tf.device(device):
    a = tf.constant([1, 2, 3])

# Print info about local cpu/gpu devices through tensorflow library
from tensorflow.python.client import device_lib

# The most useful information is found in the first two lines of the output.
# 1st line is name of device (cpu/gpu and number).
# If there were 2 cpus on the device, there would be another entry in the list under name: '/device:cpu:1'
# The second line give the memory limit in bits.
device_lib.list_local_devices()

[name: "/device:CPU:0"
 device_type: "CPU"
 memory_limit: 268435456
 locality {
 }
 incarnation: 3513264468343507029
 xla_global_id: -1]

BACK TO TOP