INFO     - The mmdet config folder already exists. No need to downloaded it. Path : /home/frederik/.icevision/mmdetection_configs/mmdetection_configs-2.20.1/configs | icevision.models.mmdet.download_configs:download_mmdet_configs:17

Test data setup

import icedata
import pickle
import shutil
from pathlib import Path

test_object_detection_data_dir = icedata.fridge.load_data()
test_object_detection_class_map = icedata.fridge.class_map()
test_object_detection_parser = icedata.fridge.parser(test_object_detection_data_dir)
test_object_detection_train_records, test_object_detection_valid_records = test_object_detection_parser.parse()

test_instance_segmentation_data_path = icedata.pennfudan.load_data()
test_instance_segmentation_parser = icedata.pennfudan.parser(data_dir=test_instance_segmentation_data_path)
test_instance_segmentation_train_records, test_instance_segmentation_valid_records = test_instance_segmentation_parser.parse()
test_instance_segmentation_class_map = test_instance_segmentation_train_records[0].detection.class_map

General

class RecordDataframeParser[source]

RecordDataframeParser(record_template) :: Parser

IceVision parser for pandas dataframes. This parser is mostly used by the RecordDataset to load records from a saved RecordDataset.

class RecordDataset[source]

RecordDataset(records:Union[List[BaseRecord], ObservableList, str], class_map, name=None, description=None) :: GenericDataset

Base class dashboard datasets that are based on IceVision records.

class ResultsDataset[source]

ResultsDataset(dataframe, name=None, description=None) :: GenericDataset

Dashboard dataset for the results of and object detection system.

Object detection

class BboxRecordDataframeParser[source]

BboxRecordDataframeParser(record_dataframe, class_map) :: RecordDataframeParser

Extends the RecordDataframeParser for object detection

class DataDescriptorBbox[source]

DataDescriptorBbox() :: DatasetDescriptor

Dashboard dataset for object detection

class StatsDescriptorBbox[source]

StatsDescriptorBbox() :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class ImageStatsDescriptorBbox[source]

ImageStatsDescriptorBbox() :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class ClassStatsDescriptorBbox[source]

ClassStatsDescriptorBbox() :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class GalleryStatsDescriptorBbox[source]

GalleryStatsDescriptorBbox() :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class BboxRecordDataset[source]

BboxRecordDataset(records:Union[List[BaseRecord], ObservableList, str], class_map=None, name=None, description=None) :: RecordDataset

Base class dashboard datasets that are based on IceVision records.

test_object_detection_record_dataset = BboxRecordDataset(test_object_detection_valid_records, test_object_detection_class_map)
assert isinstance(test_object_detection_record_dataset.data, pd.DataFrame)
assert isinstance(test_object_detection_record_dataset.stats_dataset, pd.DataFrame)
assert isinstance(test_object_detection_record_dataset.stats_class, pd.DataFrame)
assert isinstance(test_object_detection_record_dataset.stats_image, pd.DataFrame)
assert isinstance(test_object_detection_record_dataset.stats, pd.DataFrame)

assert isinstance(test_object_detection_record_dataset.__repr__(), str)
test_object_detection_split_train_records, test_object_detection_split_valid_records = test_object_detection_record_dataset.split_in_train_and_val(0.8)
assert len(test_object_detection_record_dataset.base_data) == len(test_object_detection_split_train_records) + len(test_object_detection_split_valid_records)
assert test_object_detection_record_dataset.num_images == 26

test_object_detection_added_record_dataset = test_object_detection_record_dataset + test_object_detection_record_dataset
assert len(test_object_detection_added_record_dataset) == len(test_object_detection_record_dataset)*2
assert test_object_detection_added_record_dataset.name == None
assert test_object_detection_added_record_dataset.description == None
assert test_object_detection_added_record_dataset.class_map == test_object_detection_record_dataset.class_map

test_object_detection_mask = np.array([False]*test_object_detection_record_dataset.data.shape[0])
test_object_detection_mask[1] = True
test_object_detection_masked_record = test_object_detection_record_dataset.create_new_from_mask(test_object_detection_record_dataset, test_object_detection_mask)
assert len(test_object_detection_masked_record) == 1

test_object_detection_old_record_dataset_stats = test_object_detection_record_dataset.stats_dataset
test_object_detection_record_dataset.records.list = test_object_detection_train_records
test_object_detection_new_record_dataset_stats = test_object_detection_record_dataset.stats_dataset
assert all(test_object_detection_new_record_dataset_stats != test_object_detection_old_record_dataset_stats)

test_object_detection_regenerated_record_dataset = BboxRecordDataset.load_from_record_dataframe(test_object_detection_record_dataset.data, test_object_detection_class_map)
assert len(test_object_detection_regenerated_record_dataset.records) == len(test_object_detection_record_dataset.records)
test_object_detection_regenerated_record_dataset_no_class_map = BboxRecordDataset.load_from_record_dataframe(test_object_detection_record_dataset.data)
assert sorted(test_object_detection_regenerated_record_dataset_no_class_map.class_map._id2class) == sorted(test_object_detection_class_map._id2class)

shutil.rmtree("dump_dir", ignore_errors=True)
os.mkdir("dump_dir")
test_object_detection_record_dataset.name = ""
test_object_detection_record_dataset.save("dump_dir")
test_object_detection_record_dataset.save("dump_dir")
assert len(os.listdir("dump_dir")) == 2
assert os.path.isfile("dump_dir/dataset.json")
test_object_detection_loaded_record_dataset = BboxRecordDataset("dump_dir/dataset.json")
assert test_object_detection_record_dataset.data.sort_values("area").shape == test_object_detection_loaded_record_dataset.data.sort_values("id").shape
shutil.rmtree("dump_dir")

class PrecisionRecallMetricsDescriptorObjectDetection[source]

PrecisionRecallMetricsDescriptorObjectDetection(ious=None) :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class ObjectDetectionResultsDataset[source]

ObjectDetectionResultsDataset(dataframe, name=None, description=None) :: ResultsDataset

Dashboard dataset for the results of and object detection system.

test_odrd = ObjectDetectionResultsDataset.load("test_data/object_detection_result_ds.dat")
test_object_detection_preds = pickle.load(open("test_data/object_detection_preds.pkl", "rb"))
test_object_detection_samples = pickle.load(open("test_data/object_detection_samples.pkl", "rb"))

# make the saved samples indipendent of the saved enviroment
for sample in test_object_detection_samples:
    sample.common.filepath = Path(str(test_object_detection_data_dir).split(".icevision")[0] + ".icevision" + str(sample.common.filepath).split(".icevision")[-1])

test_odrd_from_samples = ObjectDetectionResultsDataset.init_from_preds_and_samples(test_object_detection_preds, test_object_detection_samples)

Instance segmentation

class InstanceSegmentationRecordDataframeParser[source]

InstanceSegmentationRecordDataframeParser(record_dataframe, class_map) :: RecordDataframeParser

Extends the RecordDataframeParser for instance segmentation

class DataDescriptorInstanceSegmentation[source]

DataDescriptorInstanceSegmentation() :: DatasetDescriptor

Dashboard dataset for object detection

class StatsDescriptorInstanceSegmentation[source]

StatsDescriptorInstanceSegmentation() :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class ImageStatsDescriptorInstanceSegmentation[source]

ImageStatsDescriptorInstanceSegmentation() :: ImageStatsDescriptorBbox

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class ClassStatsDescriptorInstanceSegmentation[source]

ClassStatsDescriptorInstanceSegmentation() :: ClassStatsDescriptorBbox

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class GalleryStatsDescriptorInstanceSegmentation[source]

GalleryStatsDescriptorInstanceSegmentation() :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class InstanceSegmentationRecordDataset[source]

InstanceSegmentationRecordDataset(records:Union[List[BaseRecord], ObservableList, str], class_map=None, name=None, description=None) :: RecordDataset

Base class dashboard datasets that are based on IceVision records.

test_instance_segmentation_record_dataset = InstanceSegmentationRecordDataset(test_instance_segmentation_train_records, test_instance_segmentation_train_records[0].detection.class_map)
assert isinstance(test_instance_segmentation_record_dataset.data, pd.DataFrame)
assert isinstance(test_instance_segmentation_record_dataset.stats_dataset, pd.DataFrame)
assert isinstance(test_instance_segmentation_record_dataset.stats_class, pd.DataFrame)
assert isinstance(test_instance_segmentation_record_dataset.stats_image, pd.DataFrame)
assert isinstance(test_instance_segmentation_record_dataset.stats, pd.DataFrame)

test_instance_segmentation_mask = np.array([False]*test_instance_segmentation_record_dataset.data.shape[0])
test_instance_segmentation_mask[1] = True
test_instance_segmentation_masked_test_record = test_instance_segmentation_record_dataset.create_new_from_mask(test_instance_segmentation_record_dataset, test_instance_segmentation_mask)
assert len(test_instance_segmentation_masked_test_record) == 1

test_old_record_dataset_stats = test_instance_segmentation_record_dataset.stats_dataset
test_instance_segmentation_record_dataset.records.list = test_instance_segmentation_train_records
test_new_record_dataset_stats = test_instance_segmentation_record_dataset.stats_dataset
assert all(test_old_record_dataset_stats != test_new_record_dataset_stats)

test_instance_segmentation_regenerated_record_dataset = InstanceSegmentationRecordDataset.load_from_record_dataframe(test_instance_segmentation_record_dataset.data, test_instance_segmentation_class_map)
assert len(test_instance_segmentation_regenerated_record_dataset.records) == len(test_instance_segmentation_record_dataset.records)

shutil.rmtree("dump_dir", ignore_errors=True)
os.mkdir("dump_dir")
test_instance_segmentation_record_dataset.name = "dataset_instance_segmentation"
test_instance_segmentation_record_dataset.save("dump_dir")
test_instance_segmentation_record_dataset.save("dump_dir")
assert len(os.listdir("dump_dir")) == 2
assert os.path.isfile("dump_dir/dataset_instance_segmentation.json")
test_instance_segmentation_loaded_record_dataset = InstanceSegmentationRecordDataset("dump_dir/dataset_instance_segmentation.json")
assert test_instance_segmentation_record_dataset.data.sort_values("id").shape == test_instance_segmentation_loaded_record_dataset.data.sort_values("id").shape
shutil.rmtree("dump_dir")

class PrecisionRecallMetricsDescriptorInstanceSegmentation[source]

PrecisionRecallMetricsDescriptorInstanceSegmentation(ious=None) :: DatasetDescriptor

Abstract base class for descriptors of datasets. The private name of the descriptor is the defined name with a prefix _. The get function will call the calculate_description function if the value of the descriptor is None and then return the value else it will just return the value of the descriptor. The set function only allows for the attribute to be set to None, which will trigger a recomputation the next time the get function is called. When inheriting this class the function calculate_description needs to be implemented, which defines how the private value should be calculated.

class InstanceSegmentationResultsDataset[source]

InstanceSegmentationResultsDataset(dataframe, name=None, description=None) :: ResultsDataset

Dashboard dataset for the results of and object detection system.

test_instance_segmentation_odrd = InstanceSegmentationResultsDataset.load("test_data/instance_segmentation_result_ds_valid.dat")
test_instance_segmentation_preds = pickle.load(open("test_data/instance_segmentation_preds.pkl", "rb"))
test_instance_segmentation_samples = pickle.load(open("test_data/instance_segmentation_samples.pkl", "rb"))

# make the saved samples indipendent of the saved enviroment
for sample in test_instance_segmentation_samples:
    for _ in sample.components:
        if isinstance(_, FilepathRecordComponent):
            _.set_filepath(Path(str(test_instance_segmentation_data_path).split(".icevision")[0] + ".icevision" + str(sample.common.filepath).split(".icevision")[-1]))
    sample.detection.masks[0].filepath = Path(str(test_instance_segmentation_data_path).split(".icevision")[0] + ".icevision" + str(sample.common.filepath).split(".icevision")[-1])
        
test_odrd_from_samples = InstanceSegmentationResultsDataset.init_from_preds_and_samples(test_instance_segmentation_preds, test_instance_segmentation_samples)