evaluate.py

#!/usr/bin/env python

# ==============================================================================
# MIT License
#
# Copyright 2021 Institute for Automotive Engineering of RWTH Aachen University.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# ==============================================================================

import os
import tqdm
import numpy as np
import cv2
import matplotlib.pyplot as plt
import tensorflow as tf
import json

import utils
import config

conf = config.getConf()

# load network architecture module
architecture = utils.load_module(conf.model)

# get max_samples_validation random validation samples
files_input = utils.get_files_in_folder(conf.input_validation)
files_label = utils.get_files_in_folder(conf.label_validation)
_, idcs = utils.sample_list(files_label, n_samples=conf.max_samples_validation)
files_input = np.take(files_input, idcs)
files_label = np.take(files_label, idcs)
n_samples = len(files_label)
print(f"Found {n_samples} samples")

model = architecture.getModel(conf.y_min, conf.y_max, conf.x_min, conf.x_max,
                              conf.step_x_size, conf.step_y_size,
                              conf.max_points_per_pillar, conf.max_pillars,
                              conf.number_features, conf.number_channels,
                              conf.label_resize_shape, 1)
model.load_weights(conf.model_weights)
print(f"Reloaded model from {conf.model_weights}")

# evaluate
print("Evaluating ...")
eval_dir = os.path.join(os.path.dirname(conf.model_weights), os.pardir,
                        "Evaluation")

# evaluation metrics
evaluation_dict = {}
evaluation_dict['deep'] = {}
evaluation_dict['deep']['KL_distance'] = []
evaluation_dict['deep']['m_unknown'] = []
evaluation_dict['deep']['m_occupied'] = []
evaluation_dict['deep']['m_free'] = []
evaluation_dict['naive'] = {}
evaluation_dict['naive']['KL_distance'] = []
evaluation_dict['naive']['m_unknown'] = []
evaluation_dict['naive']['m_occupied'] = []
evaluation_dict['naive']['m_free'] = []


def parseSampleFn(input_file, sample_idx, label_file=None):

    # convert sample index to batch element index
    batch_element_idx = sample_idx % conf.batch_size

    # convert PCD file to matrix with columns (x, y, z, i)
    #input_file = bytes.decode(input_file)
    lidar = utils.readPointCloud(input_file, conf.intensity_threshold)

    if label_file is not None:
        # convert grid map image to matrix
        #label_file = bytes.decode(label_file)
        grid_map = tf.image.decode_image(tf.io.read_file(label_file))

    # create point pillars
    pillars, voxels = utils.make_point_pillars(
        lidar, conf.max_points_per_pillar, conf.max_pillars, conf.step_x_size,
        conf.step_y_size, conf.x_min, conf.x_max, conf.y_min, conf.y_max,
        conf.z_min, conf.z_max, min_distance = conf.min_point_distance)
    pillars = pillars.astype(np.float32)
    voxels = voxels.astype(np.int32)
    voxels[..., 0] = batch_element_idx

    # convert grid map to tensorflow label
    if label_file is not None:
        grid_map = tf.image.resize(
            grid_map,
            conf.label_resize_shape[0:2],
            method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
        # use only channels 'free' and 'occupied'
        grid_map = tf.cast(grid_map[..., 1:3], tf.float32)
        # normalize from image [0..255] to [0.0..1.0]
        grid_map = tf.divide(grid_map, 255.0)

    network_inputs = (pillars, voxels)
    if label_file is not None:
        network_labels = (grid_map)
    else:
        network_labels = None

    return network_inputs, network_labels


for k in tqdm.tqdm(range(n_samples)):

    input_file = files_input[k]
    label_file = files_label[k]

    input, label = parseSampleFn(input_file, 0, label_file)
    prediction = model.predict(input).squeeze()

    sample_name = os.path.splitext(os.path.basename(input_file))[0]

    kld = tf.keras.metrics.KLDivergence()

    # collect belief masses and Kullback-Leibler distance for predictions by deep ISM
    prob, u, _, _ = utils.evidences_to_masses(prediction)
    evaluation_dict['deep']['m_unknown'].append(float(tf.reduce_mean(u)))
    evaluation_dict['deep']['m_free'].append(
        float(tf.reduce_mean(prob[..., 0])))
    evaluation_dict['deep']['m_occupied'].append(
        float(tf.reduce_mean(prob[..., 1])))
    evaluation_dict['deep']['KL_distance'].append(float(kld(label,
                                                            prediction)))

    # save predicted grid map
    prediction_dir = os.path.join(eval_dir, "predictions")
    if not os.path.exists(prediction_dir):
        os.makedirs(prediction_dir)
    prediction_img = utils.evidence_to_ogm(prediction)
    output_file = os.path.join(prediction_dir,
                               os.path.basename(files_input[k]))
    cv2.imwrite(output_file + ".png",
                cv2.cvtColor(prediction_img, cv2.COLOR_RGB2BGR))

    # save input point cloud as image
    if conf.store_input_point_clouds:
        input_dir = os.path.join(eval_dir, "inputs")
        if not os.path.exists(input_dir):
            os.makedirs(input_dir)
        lidar = utils.readPointCloud(input_file, conf.intensity_threshold)
        lidar_bev = utils.lidar_to_bird_view_img(lidar,
                                                conf.x_min,
                                                conf.x_max,
                                                conf.y_min,
                                                conf.y_max,
                                                conf.step_x_size,
                                                conf.step_y_size,
                                                factor=2)
        output_file = os.path.join(input_dir, os.path.basename(files_input[k]))
        cv2.imwrite(output_file + ".png", cv2.cvtColor(lidar_bev,
                                                    cv2.COLOR_RGB2BGR))

    # save label as image
    if conf.store_labels:
        label_img = utils.evidence_to_ogm(label)
        label_dir = os.path.join(eval_dir, "labels")
        if not os.path.exists(label_dir):
            os.makedirs(label_dir)
        output_file = os.path.join(label_dir, os.path.basename(files_input[k]))
        cv2.imwrite(output_file + ".png", cv2.cvtColor(label_img,
                                                    cv2.COLOR_RGB2BGR))

    # create "naive" occupancy grid map for comparision
    naive_ogm = utils.naive_geometric_ISM(input_file, conf.x_min, conf.x_max,
                                          conf.y_min, conf.y_max,
                                          conf.step_x_size, conf.step_y_size,
                                          -1.11, 0.39, conf.min_point_distance)
    naive_ogm = cv2.resize(
        naive_ogm, (conf.label_resize_shape[1], conf.label_resize_shape[0]))

    if conf.store_naive_ogms:
        naive_ogm_dir = os.path.join(eval_dir, "naive_ogm")
        if not os.path.exists(naive_ogm_dir):
            os.makedirs(naive_ogm_dir)
        naive_ogm_file = os.path.join(naive_ogm_dir, sample_name + '.png')
        cv2.imwrite(naive_ogm_file, naive_ogm)

    # collect belief masses and Kullback-Leibler distance for OGMs by geometric ISM
    naive_ogm = naive_ogm.astype(
        np.float32
    )[..., 1:
      3] / 255.0  # convert green and red layer to evidence layers for free and occupied
    prob_naive, u_naive, _, _ = utils.evidences_to_masses(naive_ogm)
    evaluation_dict['naive']['m_unknown'].append(float(
        tf.reduce_mean(u_naive)))
    evaluation_dict['naive']['m_free'].append(
        float(tf.reduce_mean(prob_naive[..., 0])))
    evaluation_dict['naive']['m_occupied'].append(
        float(tf.reduce_mean(prob_naive[..., 1])))
    evaluation_dict['naive']['KL_distance'].append(float(kld(label,
                                                             naive_ogm)))

# create subfolders
plot_dir = os.path.join(eval_dir, "plots")
raw_dir = os.path.join(eval_dir, "raw")
if not os.path.exists(plot_dir):
    os.makedirs(plot_dir)
if not os.path.exists(raw_dir):
    os.makedirs(raw_dir)

# plot cross entropy over evaluation dataset
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_xlabel('time in seconds')

# Turn off axis lines and ticks of the big subplot
ax.spines['top'].set_color('none')
ax.spines['bottom'].set_color('none')
ax.spines['left'].set_color('none')
ax.spines['right'].set_color('none')
ax.tick_params(labelcolor='w',
               top=False,
               bottom=False,
               left=False,
               right=False)

ax1 = fig.add_subplot(211)
ax2 = fig.add_subplot(212)

t = np.arange(0, len(evaluation_dict['naive']['m_unknown']))
ax1.plot(t, evaluation_dict['deep']['m_unknown'], 'b-', t,
         evaluation_dict['deep']['m_free'], 'g-', t,
         evaluation_dict['deep']['m_occupied'], 'r-', t,
         evaluation_dict['naive']['m_unknown'], 'b--', t,
         evaluation_dict['naive']['m_free'], 'g--', t,
         evaluation_dict['naive']['m_occupied'], 'r--')
ax1.set_ylim(0, 1.0)
ax1.legend([
    r'$\overline{m}(\Theta)$', r'$\overline{m}(F)$', r'$\overline{m}(O)$',
    r'$\overline{m}_G(\Theta)$', r'$\overline{m}_G(F)$', r'$\overline{m}_G(O)$'
])

ax2.plot(t, evaluation_dict['deep']['KL_distance'], 'k-', t,
         evaluation_dict['naive']['KL_distance'], 'k--')
ax2.legend([
    r'$KL\left[Dir(p|\hat{\alpha})||Dir(p|\alpha)\right]$',
    r'$KL\left[Dir(p|\hat{\alpha}_G)||Dir(p|\alpha)\right]$'
])

plt.savefig(os.path.join(plot_dir, 'evaluation.png'))

# store values as json file
evaluation_json = dict()
evaluation_json['eval_kld'] = np.vstack(
    (t, evaluation_dict['deep']['KL_distance'])).transpose().tolist()
evaluation_json['eval_uncertainty'] = np.vstack(
    (t, evaluation_dict['deep']['m_unknown'])).transpose().tolist()
evaluation_json['eval_prob_free'] = np.vstack(
    (t, evaluation_dict['deep']['m_free'])).transpose().tolist()
evaluation_json['eval_prob_occupied'] = np.vstack(
    (t, evaluation_dict['deep']['m_occupied'])).transpose().tolist()

evaluation_json['eval_naive_kld'] = np.vstack(
    (t, evaluation_dict['naive']['KL_distance'])).transpose().tolist()
evaluation_json['eval_naive_uncertainty'] = np.vstack(
    (t, evaluation_dict['naive']['m_unknown'])).transpose().tolist()
evaluation_json['eval_naive_prob_free'] = np.vstack(
    (t, evaluation_dict['naive']['m_free'])).transpose().tolist()
evaluation_json['eval_naive_prob_occupied'] = np.vstack(
    (t, evaluation_dict['naive']['m_occupied'])).transpose().tolist()
with open(os.path.join(raw_dir, 'evaluation.json'), 'w') as fp:
    json.dump(evaluation_json, fp)