NIST Special Database3
- 具体指的是一个更大的特殊数据库3；
- 该数据库的内容为手写数字黑白图片；
- 该数据库由美国人口普查局的雇员手写
NIST Special Database1
- 特殊数据库1；
- 该数据库的内容为手写数字黑白图片；
- 该数据库的图片由高中学生手写；
MNIST
- MNIST 数据库：Modified National Institute of Standards and Technology 数据库
- 是一个大的手写数字的集合；
- 具有训练集60,000个；
- 测试集10,000个；
- 是NIST3和NIST1的子集；
- 数字图片已经被居中，以固定的尺寸值标准化处理；
- 原始的黑白两层图像被设置为20x20 像素大小，且保持宽高比；
- 结果图像在标准化算法中的反走样技术的处理下包含灰度级图像；
- 通过计算像素的质心，和平移操作，手写的数字被居中放置到尺寸为28X28的图片中；
MNIST 用法
transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize([0,],[1,])])
trainset=torchvision.datasets.MNIST(root='./data',train=True,download=True,transform=transform)
trainloader=torch.utils.data.DataLoader(trainset,batch_size=32,shuffle=True,num_workers=2)
testset=torchvision.datasets.MNIST(root='./data',train=False,download=True,transform=transform)
testloader=torch.utils.data.DataLoader(testset,batch_size=32,shuffle=True,num_workers=2)
MNIST 源码（python）
import codecs
import os
import os.path
import shutil
import string
import sys
import warnings
from typing import Any,Callable,Dict,List,Optional,Tuple
from urllib.error import URLErrorimport numpy as np
import torch
from PIL import Imagefrom .utils import _flip_byte_order,check_integrity,download_and_extract_archive,extract_archive,verify_str_arg
from .vision import VisionDatasetclass MNIST(VisionDataset):''''MNIST <http://yann.lecun.com/exdb/mnist/>' _Dataset.'''mirrors=["http://yann.lecun.com/exdb/mnist/","https://ossci-datasets.s3.amazonaws.com/mnist/"]resource=[("train-images-idx3-ubyte.gz","f68b3c2dcbeaaa9fbdd348bbdeb94873"),("train-labels-idx1-ubyte.gz","d53e105ee54ea40749a09fcbcd1e9432"),("t10k-images-idx3-ubyte.gz","9fb629c4189551a2d022fa330f9573f3"),("t10k-labels-idx1-ubyte.gz","ec29112dd5afa0611ce80d1b7f02629c")]training_file="training.pt"test_file="test.pt"classes=["0-zero","1-one","2-two","3-three","4-four","5-five","6-six","7-seven","8-eight","9-nine"]@propertydef train_labels(self):warnings.warn("train_labels has been renamed targets")return self.targets@propertydef test_labels(self):warnings.warn("test_labels has been renamed targets")return self.targets@propertydef train_data(self):warnings.warn("train_data has been renamed data")return self.data@propertydef test_data(self):warnings.warn("test_data has been renamed data")return self.datadef __init__(self,root:str,train:bool=True,transform:Optional[Callable]=None,target_transform:Optional[Callable]=None,download:bool=False)->None:'''Args:param root: string,root directory of dataset where 'MNIST/raw/train-images-idx3-ubyte' and 'MNIST/raw/t10k-images-idx3-ubyte' exist.:param train:(bool,optional),if true,creates dataset from 'train-images-idx3-utyte',otherwise from 't10k-images-idx3-utyte'.:param transform:(callable,optional),a function/transform that takes in an PIL image and returns a transformed version.E.g,'transform.RandomCrop':param target_transform:(callable,optional),a function/transform that takes in the target and transform it.:param download:(bool,optional),if True,downloads the dataset from the internet and puts it in root directory.If dataset is already downloaded,it is not download again.'''super().__init__(root,transform,target_transform)self.train=trainif self._check_legacy_exist():self.data,self.targets=self._load_legacy_data()returnif download:self.download()if not self._check_exists():raise RuntimeError("Dataset not found.You can use download=True to download it")self.data,self.targets=self._load_data()def _check_legacy_exist(self):processed_folder_exists=os.path.exists(self.processed_folder)if not processed_folder_exists:return Falsereturn all(check_integrity(os.path.join(self.processed_folder,file)) for file in (self.training_file,self.test_file))def _load_legacy_data(self):#This is for BC only,We no longer cache the data in a custom binary,but simply read from the raw data directly.data_file=self.training_file if self.train else self.test_filereturn torch.load(os.path.join(self.processed_folder,data_file))def _load_data(self):image_file = f"{'train' if self.train else 't10k'}-images-idx3-ubyte"data = read_image_file(os.path.join(self.raw_folder, image_file))label_file = f"{'train' if self.train else 't10k'}-labels-idx1-ubyte"targets = read_label_file(os.path.join(self.raw_folder, label_file))return data, targetsdef __getitem__(self, index: int) -> Tuple[Any, Any]:"""Args:index (int): IndexReturns:tuple: (image, target) where target is index of the target class."""img, target = self.data[index], int(self.targets[index])# doing this so that it is consistent with all other datasets# to return a PIL Imageimg = Image.fromarray(img.numpy(), mode="L")if self.transform is not None:img = self.transform(img)if self.target_transform is not None:target = self.target_transform(target)return img, targetdef __len__(self) -> int:return len(self.data)@propertydef raw_folder(self) -> str:return os.path.join(self.root, self.__class__.__name__, "raw")@propertydef processed_folder(self) -> str:return os.path.join(self.root, self.__class__.__name__, "processed")@propertydef class_to_idx(self) -> Dict[str, int]:return {_class: i for i, _class in enumerate(self.classes)}def _check_exists(self) -> bool:return all(check_integrity(os.path.join(self.raw_folder, os.path.splitext(os.path.basename(url))[0]))for url, _ in self.resources)def download(self) -> None:"""Download the MNIST data if it doesn't exist already."""if self._check_exists():returnos.makedirs(self.raw_folder, exist_ok=True)# download filesfor filename, md5 in self.resources:for mirror in self.mirrors:url = f"{mirror}{filename}"try:print(f"Downloading {url}")download_and_extract_archive(url, download_root=self.raw_folder, filename=filename, md5=md5)except URLError as error:print(f"Failed to download (trying next):\n{error}")continuefinally:print()breakelse:raise RuntimeError(f"Error downloading {filename}")def extra_repr(self) -> str:split = "Train" if self.train is True else "Test"return f"Split: {split}"
