kitti数据集简介

kitti数据集是比较早出来的3D检测方面的数据集，相对来说数据结构简单，适合做单目检测的工作，目前也是业界和学术界常用的公开数据集。
自己最近也在做单目3D检测的工作，所以也分享一些理解，希望能给到一些人启发和帮助。
kitti数据的目录结构如下：
只需要下载下图所示的四个zip包解压就行，做单目3D检测已经足够。

解压后以training为例，其中calib里面是传感器的标定参数，有一堆txt文件，内容格式如下：

P0: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 0.000000000000e+00 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 0.000000000000e+00 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 0.000000000000e+00
P1: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 -3.875744000000e+02 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 0.000000000000e+00 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 0.000000000000e+00
P2: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 4.485728000000e+01 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 2.163791000000e-01 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 2.745884000000e-03
P3: 7.215377000000e+02 0.000000000000e+00 6.095593000000e+02 -3.395242000000e+02 0.000000000000e+00 7.215377000000e+02 1.728540000000e+02 2.199936000000e+00 0.000000000000e+00 0.000000000000e+00 1.000000000000e+00 2.729905000000e-03
R0_rect: 9.999239000000e-01 9.837760000000e-03 -7.445048000000e-03 -9.869795000000e-03 9.999421000000e-01 -4.278459000000e-03 7.402527000000e-03 4.351614000000e-03 9.999631000000e-01
Tr_velo_to_cam: 7.533745000000e-03 -9.999714000000e-01 -6.166020000000e-04 -4.069766000000e-03 1.480249000000e-02 7.280733000000e-04 -9.998902000000e-01 -7.631618000000e-02 9.998621000000e-01 7.523790000000e-03 1.480755000000e-02 -2.717806000000e-01
Tr_imu_to_velo: 9.999976000000e-01 7.553071000000e-04 -2.035826000000e-03 -8.086759000000e-01 -7.854027000000e-04 9.998898000000e-01 -1.482298000000e-02 3.195559000000e-01 2.024406000000e-03 1.482454000000e-02 9.998881000000e-01 -7.997231000000e-01

p0到p4对应的是rect相机坐标系到像素坐标系的变换矩阵，一般论文中叫内参，实际上是相机内参和相机外参以及畸变矫正融合的矩阵。
这里提几点：
kitti有四个相机，两个彩色的，两个灰度的，左侧彩色(对应序号2)的是有对应的标注信息的，所以如果做单目的话，用p2。
rect相机坐标系是对参考坐标系进行旋转矫正，使得4个相机的光心共面，这样做有一定好处，具体好处是什么可以不用在意。
Tr_velo_to_cam是雷达坐标系到参考相机坐标系。
Tr_imu_to_velo是imu到雷达坐标系。
image_2里面就是图片了。
label_2里面就是对应的标注信息了，3D标注是在rect相机坐标系中的。

Pedestrian 0.00 0 -0.20 712.40 143.00 810.73 307.92 1.89 0.48 1.20 1.84 1.47 8.41 0.01

第一列是类别，第二列是截断程度(0~1)，第三列是遮挡程度(0,1,2,3)，第四列是观察者角度alpha(没啥用)，接下来五列到八列是2d框的信息，xmin ymin xmax ymax四个坐标，九列到十一列是3D框的高宽长，十二到十四列是3D框下底面中心点，最后一列是绕y轴的旋转角。
velodyne里面是雷达。另外一个velodyne_reduced是mmdetection3d转数据格式时生成的，可以不在意。

可视化

main.py文件：

import os
import cv2
import numpy as np
import mayavi.mlab as mlab
from utils import draw_lidar, draw_2dbox, gen_3dbox, project_box3d, draw_project, draw_box3d_lidar


class Kitti:
    def __init__(self, root_path="./kitti", ind=0) -> None:
        self.root_path = root_path
        train_file = os.path.join(root_path, "ImageSets/train.txt")
        with open(train_file, 'r') as f:
            names = f.readlines()
        self.names = [name.rstrip() for name in names]
        self.name = self.names[ind]

    def get_image(self, show=False):
        img_path = os.path.join(self.root_path, "training/image_2", self.name+".png")
        img = cv2.imread(img_path)
        if show and os.path.exists(img_path):
            cv2.imshow("origin image", img)
            if cv2.waitKey(0) == ord("q"):
                cv2.destroyAllWindows()
        return img

    def get_lidar(self, show=False):
        lidar_path = os.path.join(self.root_path, "training/velodyne", self.name+".bin")
        lidar = np.fromfile(lidar_path, dtype=np.float32)
        lidar = lidar.reshape((-1, 4))
        if show:
            fig = mlab.figure(figure=None, bgcolor=(0, 0, 0), engine=None,
                            size=(1000, 500))
            draw_lidar(lidar, fig=fig)
            mlab.show()
        return lidar
    
    def get_calib(self):
        calib = {}
        calib_path = os.path.join(self.root_path, "training/calib", self.name+".txt")
        with open(calib_path, 'r') as cf:
            infos = cf.readlines()
            infos = [x.rstrip() for x in infos]
        for info in infos:
            if len(info) == 0:
                continue
            key, value = info.split(":", 1)
            calib[key] = np.array([float(x) for x in value.split()])
        calib_format = self.format_calib(calib)
        return calib_format

    def format_calib(self, calib):
        calib_format = {}
        # projection matrix from rect coord to image coord.
        rect2image = calib["P2"]
        rect2image = rect2image.reshape([3, 4])
        calib_format["rect2image"] = rect2image
        # projection matrix from lidar coord to reference cam coord.
        lidar2cam = calib["Tr_velo_to_cam"]
        lidar2cam = lidar2cam.reshape([3, 4])
        calib_format["lidar2cam"] = lidar2cam
        # projection matrix from rect cam coord to reference cam coord.
        rect2ref = calib["R0_rect"]
        rect2ref = rect2ref.reshape([3, 3])
        calib_format["rect2ref"] = rect2ref
        return calib_format

    def get_anns(self):
        anns = []
        label_path = os.path.join(self.root_path, "training/label_2", self.name+".txt")
        with open(label_path, 'r') as lf:
            labels = lf.readlines()
            labels = [label.rstrip() for label in labels]
        for label in labels:
            ann_format = {}
            ann = label.split(" ")
            class_name = ann[0]
            ann_format["class_name"]=class_name
            ann_ = [float(x) for x in ann[1:]]
            truncation = ann_[0] # truncated pixel ratio [0..1]
            ann_format["truncation"]=truncation
            occlusion = ann_[1] # 0=visible, 1=partly occluded, 2=fully occluded, 3=unknown
            ann_format["occlusion"]=occlusion
            alpha = ann_[2]
            ann_format["alpha"]=alpha # object observation angle [-pi..pi]

            #2D box
            xmin, ymin, xmax, ymax = ann_[3], ann_[4], ann_[5], ann_[6]
            box2d = np.array([xmin, ymin, xmax, ymax])
            ann_format["box2d"]=box2d

            #3D box
            box3d = {}
            h, w, l = ann_[7], ann_[8], ann_[9]
            cx, cy, cz = ann_[10], ann_[11], ann_[12]
            box3d["dim"] = np.array([l, w, h])
            box3d["center"] = np.array([cx, cy, cz])
            yaw = ann_[13]
            box3d["rotation"] = yaw# yaw angle [-pi..pi]
            ann_format["box3d"]=box3d

            anns.append(ann_format)
        return anns


class VisKitti:
    def __init__(self, root_path="./kitti", ind=0) -> None:
        self.kitti = Kitti(root_path=root_path, ind=ind)
        self.calib = self.kitti.get_calib()
        self.anns = self.kitti.get_anns()

    def show_origin_image(self):
        self.kitti.get_image(show=True)

    def show_origin_lidar(self):
        self.kitti.get_lidar(show=True)

    def show_image_with_2dbox(self, save=False):
        img = self.kitti.get_image()
        bbox = []
        names = []
        for ann in self.anns:
            bbox.append(ann["box2d"])
            names.append(ann["class_name"])
        draw_2dbox(img, bbox, names, save=save)
        

    def show_image_with_project_3dbox(self, show=True):
        img = self.kitti.get_image()
        bbox = []
        for ann in self.anns:
            bbox.append(ann["box3d"])
        bbox3d = gen_3dbox(bbox3d=bbox)
        project_xy,_ = project_box3d(bbox3d, self.calib)
        draw_project(img, project_xy, save=False)
        

    def show_lidar_with_3dbox(self, img_fov=False):
        img = self.kitti.get_image()
        bbox = []
        for ann in self.anns:
            bbox.append(ann["box3d"])
        bbox3d = gen_3dbox(bbox3d=bbox)
        fig = mlab.figure(figure=None, bgcolor=(0, 0, 0), engine=None,
                        size=(1000, 500))
        lidar = self.kitti.get_lidar()
        fig = draw_lidar(lidar, fig=fig)
        fig = draw_box3d_lidar(bbox3d, self.calib, fig)
        mlab.show()     



if __name__ == "__main__":
    vis = VisKitti()
    print("1: show_origin_image")
    print("2: show_origin_lidar")
    print("3: show_image_with_2dbox")
    print("4: show_image_with_project_3dbox")
    print("5: show_lidar_with_3dbox")
    
    choice = input("please choice number:")
    if choice=="1":
        vis.show_origin_image()
    elif choice=="2":
        vis.show_origin_lidar()
    elif choice=="3":
        vis.show_image_with_2dbox()
    elif choice=="4":
        vis.show_image_with_project_3dbox()
    elif choice=="5":
        vis.show_lidar_with_3dbox()

utils.py 文件

import os
import cv2
import numpy as np
import mayavi.mlab as mlab


def draw_lidar(pc,
               color=None,
               fig=None,
               bgcolor=(0, 0, 0),
               pts_scale=0.3,
               pts_mode="point",
               pts_color=None,
               color_by_intensity=False,
               ):
    if fig == None:
        fig = mlab.figure(figure=None, bgcolor=bgcolor, engine=None,
                          size=(1600, 1000))
    if color == None:
        color = pc[:, 2]

    s = pc[:, 3] if color_by_intensity else pc[:, 2]
    mlab.points3d(pc[:, 0],
                  pc[:, 1],
                  pc[:, 2],
                  s,
                  color=pts_color,
                  mode=pts_mode,
                  colormap="gnuplot",
                  scale_factor=pts_scale,
                  figure=fig)

    # draw origin point
    mlab.points3d(0, 0, 0, color=(1, 1, 1), mode="sphere", scale_factor=0.2, figure=fig)

    # draw axis
    axis_xyz = np.array([[2, 0, 0],[0, 2, 0],[0, 0, 2]], dtype=np.float64)
    # x axis
    mlab.plot3d([0, axis_xyz[0, 0]],
                [0, axis_xyz[0, 1]],
                [0, axis_xyz[0, 2]],
                color=(1, 0, 0),
                tube_radius=None,
                figure=fig)
    # y axis
    mlab.plot3d([0, axis_xyz[1, 0]],
                [0, axis_xyz[1, 1]],
                [0, axis_xyz[1, 2]],
                color=(0, 1, 0),
                tube_radius=None,
                figure=fig)
    # z axis
    mlab.plot3d([0, axis_xyz[2, 0]],
                [0, axis_xyz[2, 1]],
                [0, axis_xyz[2, 2]],
                color=(0, 0, 1),
                tube_radius=None,
                figure=fig)
    mlab.view(
        azimuth=180,
        elevation=70,
        focalpoint=[12.0909996, -1.04700089, -2.03249991],
        distance=62.0,
        figure=fig,
    )
    return fig

def draw_2dbox(img, bbox, names=None, save=False):
    assert len(bbox)==len(names), "names not match bbox"
    color_map = {"Car":(0, 255, 0), "Pedestrian":(255, 0, 0), "Cyclist":(0, 0, 255)}
    for i, box in enumerate(bbox):
        name = names[i]
        if name not in color_map.keys():
            continue
        color = color_map[name]
        cv2.rectangle(
            img,
            (int(box[0]), int(box[1])),
            (int(box[2]), int(box[3])),
            color,
            2,
        )
        name_coord = (int(box[0]), int(max(box[1]-5, 0)))
        cv2.putText(img, name, name_coord, 
                    cv2.FONT_HERSHEY_PLAIN, 1, color, 1)

    cv2.imshow("image_with_2dbox", img)
    if cv2.waitKey(0) == ord("q"):
        cv2.destroyAllWindows()
    if save:
        cv2.imwrite("image_with_2dbox.jpg", img)

def rotx(t):
    """ 3D Rotation about the x-axis. """
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[1, 0, 0], [0, c, -s], [0, s, c]])


def roty(t):
    """ Rotation about the y-axis. """
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[c, 0, s], [0, 1, 0], [-s, 0, c]])


def rotz(t):
    """ Rotation about the z-axis. """
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[c, -s, 0], [s, c, 0], [0, 0, 1]])

def expand_matrix(matrix):
    new_matrix = np.eye(4, 4)
    new_matrix[:3, :] = matrix
    return new_matrix

def gen_3dbox(bbox3d):
    corners_3d_all = []
    for box in bbox3d:
        center = box["center"]
        l, w, h = box["dim"]
        angle = box["rotation"]
        R = roty(angle)
        # 3d bounding box corners
        x_corners = [l / 2, l / 2, -l / 2, -l / 2, l / 2, l / 2, -l / 2, -l / 2]
        y_corners = [0, 0, 0, 0, -h, -h, -h, -h]
        z_corners = [w / 2, -w / 2, -w / 2, w / 2, w / 2, -w / 2, -w / 2, w / 2]

        corners = np.vstack([x_corners, y_corners, z_corners])
        corners_3d = np.dot(R, corners)
        corners_3d[0, :] += center[0]
        corners_3d[1, :] += center[1]
        corners_3d[2, :] += center[2]
        corners_3d_all.append(corners_3d)
    return corners_3d_all

def project_box3d(bbox3d, calib):
    P = calib["rect2image"]
    P = expand_matrix(P)
    project_xy = []
    project_z = []
    for box3d in bbox3d:
        if np.any(box3d[:, 2] < 0.1):
            continue
        box3d = np.concatenate([box3d, np.zeros((1, 8))], axis=0)
        project_3dbox = np.dot(P, box3d)[:3, :]
        pz = project_3dbox[2, :]
        px = project_3dbox[0, :]/pz
        py = project_3dbox[1, :]/pz
        xy = np.stack([px, py], axis=1)
        project_xy.append(xy)
        project_z.append(pz)
    return project_xy, project_z

def draw_project(img, project_xy, save=False):
    color_map = {"Car":(0, 255, 0), "Pedestrian":(255, 0, 0), "Cyclist":(0, 0, 255)}
    for i, qs in enumerate(project_xy):
        color = (0, 255, 0)
        qs = qs.astype(np.int32)
        for k in range(0, 4):
            i, j = k, (k + 1) % 4
            # use LINE_AA for opencv3
            cv2.line(img, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, 1)
            i, j = k + 4, (k + 1) % 4 + 4
            cv2.line(img, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, 1)
            i, j = k, k + 4
            cv2.line(img, (qs[i, 0], qs[i, 1]), (qs[j, 0], qs[j, 1]), color, 1)

    cv2.imshow("image_with_projectbox", img)
    if cv2.waitKey(0) == ord("q"):
        cv2.destroyAllWindows()
    if save:
        cv2.imwrite("image_with_projectbox.jpg", img)

def draw_gt_boxes3d(
    gt_boxes3d,
    fig,
    color=(1, 1, 1),
    line_width=1,
):
    """ Draw 3D bounding boxes
    Args:
        gt_boxes3d: numpy array (n,8,3) for XYZs of the box corners
        fig: mayavi figure handler
        color: RGB value tuple in range (0,1), box line color
        line_width: box line width
        draw_text: boolean, if true, write box indices beside boxes
        text_scale: three number tuple
        color_list: a list of RGB tuple, if not None, overwrite color.
    Returns:
        fig: updated fig
    """
    num = len(gt_boxes3d)
    for n in range(num):
        b = gt_boxes3d[n]

        for k in range(0, 4):
            i, j = k, (k + 1) % 4
            mlab.plot3d(
                [b[i, 0], b[j, 0]],
                [b[i, 1], b[j, 1]],
                [b[i, 2], b[j, 2]],
                color=color,
                tube_radius=None,
                line_width=line_width,
                figure=fig,
            )

            i, j = k + 4, (k + 1) % 4 + 4
            mlab.plot3d(
                [b[i, 0], b[j, 0]],
                [b[i, 1], b[j, 1]],
                [b[i, 2], b[j, 2]],
                color=color,
                tube_radius=None,
                line_width=line_width,
                figure=fig,
            )

            i, j = k, k + 4
            mlab.plot3d(
                [b[i, 0], b[j, 0]],
                [b[i, 1], b[j, 1]],
                [b[i, 2], b[j, 2]],
                color=color,
                tube_radius=None,
                line_width=line_width,
                figure=fig,
            )
    return fig

def inverse_rigid_trans(Tr):
    """ Inverse a rigid body transform matrix (3x4 as [R|t])
        [R'|-R't; 0|1]
    """
    inv_Tr = np.zeros_like(Tr)  # 3x4
    inv_Tr[0:3, 0:3] = np.transpose(Tr[0:3, 0:3])
    inv_Tr[0:3, 3] = np.dot(-np.transpose(Tr[0:3, 0:3]), Tr[0:3, 3])
    return inv_Tr

def draw_box3d_lidar(bbox3d, calib, fig):
    # method 1
    # lidar2cam = calib["lidar2cam"]
    # lidar2cam = expand_matrix(lidar2cam)
    # cam2rect_ = calib["rect2ref"]
    # cam2rect = np.eye(4, 4)
    # cam2rect[:3, :3] = cam2rect_
    # lidar2rec = np.dot(lidar2cam, cam2rect)
    # rec2lidar = np.linalg.inv(lidar2rec) #(AB)-1 = B-1@A-1

    # method 2
    lidar2cam_ = calib["lidar2cam"]
    cam2rect_ = calib["rect2ref"]
    cam2rect = np.eye(4, 4)
    cam2rect[:3, :3] = cam2rect_
    
    # lidar2cam = np.eye(4, 4)
    # lidar2cam[:3, :] = lidar2cam_
    # cam2lidar = np.linalg.inv(lidar2cam)
    cam2lidar_ = inverse_rigid_trans(lidar2cam_)
    cam2lidar = np.eye(4, 4)
    cam2lidar[:3, :] = cam2lidar_

    all_lidar_box3d = []
    for box3d in bbox3d:
        if np.any(box3d[:, 2] < 0.1):
            continue
        box3d = np.concatenate([box3d, np.ones((1, 8))], axis=0)
        box3d_in_refcam = np.dot(np.linalg.inv(cam2rect), box3d)
        lidar_box3d = np.dot(cam2lidar, box3d_in_refcam)[:3, :]
        lidar_box3d = np.transpose(lidar_box3d)
        all_lidar_box3d.append(lidar_box3d)
    print(all_lidar_box3d)
    fig = draw_gt_boxes3d(all_lidar_box3d, fig)
    return fig

可视化结果：

总结

本来是想一行行解释代码的，但是太累，所以就不解释了，自己代码是简化过的，应该很容易看懂。主要参考的kitti_object_vis。