from torch.utils.data import TensorDataset,DataLoader
import torch
?
a = torch.tensor([[1, 2, 3],
? ? ? ? ? ? ? ? ? [4, 5, 6],
? ? ? ? ? ? ? ? ? [7, 8, 9],
? ? ? ? ? ? ? ? ? [1, 2, 3],
? ? ? ? ? ? ? ? ? [4, 5, 6],
? ? ? ? ? ? ? ? ? [7, 8, 9],
? ? ? ? ? ? ? ? ? [1, 2, 3],
? ? ? ? ? ? ? ? ? [4, 5, 6],
? ? ? ? ? ? ? ? ? [7, 8, 9],
? ? ? ? ? ? ? ? ? [1, 2, 3],
? ? ? ? ? ? ? ? ? [4, 5, 6],
? ? ? ? ? ? ? ? ? [7, 8, 9]])
?
b = torch.tensor([44, 55, 66, 44, 55, 66, 44, 55, 66, 44, 55, 66])
train_ids = TensorDataset(a,b)
# 切片輸出
print(train_ids[0:4]) # 第0,1,2,3行
# 循環(huán)取數據
for x_train,y_label in train_ids:
? ? print(x_train,y_label)

# 參數說明，dataset=train_ids表示需要封裝的數據集，batch_size表示一次取幾個
# shuffle表示亂序取數據，設為False表示順序取數據，True表示亂序取數據
train_loader = DataLoader(dataset=train_ids,batch_size=4,shuffle=False)
# 注意enumerate返回值有兩個,一個是序號，一個是數據（包含訓練數據和標簽）
for i,data in enumerate(train_loader,1):
? ? train_data, label = data
? ? print(' batch:{0} train_data:{1} ?label: {2}'.format(i+1, train_data, label))

class TensorDataset(Dataset[Tuple[Tensor, ...]]):
? ? r"""Dataset wrapping tensors.
? ? Each sample will be retrieved by indexing tensors along the first dimension.
? ? Arguments:
? ? ? ? *tensors (Tensor): tensors that have the same size of the first dimension.
? ? """
? ? tensors: Tuple[Tensor, ...]
?
? ? def __init__(self, *tensors: Tensor) -> None:
? ? ? ? assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
? ? ? ? self.tensors = tensors
?
? ? def __getitem__(self, index):
? ? ? ? return tuple(tensor[index] for tensor in self.tensors)
?
? ? def __len__(self):
? ? ? ? return self.tensors[0].size(0)
?
# 由于此類內容過多，故僅列舉了與本文相關的參數，其余參數可以自行去查看源碼
class DataLoader(Generic[T_co]):
? ? r"""
? ? Data loader. Combines a dataset and a sampler, and provides an iterable over
? ? the given dataset.
? ? The :class:`~torch.utils.data.DataLoader` supports both map-style and
? ? iterable-style datasets with single- or multi-process loading, customizing
? ? loading order and optional automatic batching (collation) and memory pinning.
? ? See :py:mod:`torch.utils.data` documentation page for more details.
? ? Arguments:
? ? ? ? dataset (Dataset): dataset from which to load the data.
? ? ? ? batch_size (int, optional): how many samples per batch to load
? ? ? ? ? ? (default: ``1``).
? ? ? ? shuffle (bool, optional): set to ``True`` to have the data reshuffled
? ? ? ? ? ? at every epoch (default: ``False``).
? ? """
? ? dataset: Dataset[T_co]
? ? batch_size: Optional[int]
?
? ? def __init__(self, dataset: Dataset[T_co], batch_size: Optional[int] = 1,
? ? ? ? ? ? ? ? ?shuffle: bool = False):
?
? ? ? ? self.dataset = dataset
? ? ? ? self.batch_size = batch_size

class Dataset(object):
? ? r"""An abstract class representing a :class:`Dataset`.

? ? All datasets that represent a map from keys to data samples should subclass
? ? it. All subclasses should overwrite :meth:`__getitem__`, supporting fetching a
? ? data sample for a given key. Subclasses could also optionally overwrite
? ? :meth:`__len__`, which is expected to return the size of the dataset by many
? ? :class:`~torch.utils.data.Sampler` implementations and the default options
? ? of :class:`~torch.utils.data.DataLoader`.

? ? .. note::
? ? ? :class:`~torch.utils.data.DataLoader` by default constructs a index
? ? ? sampler that yields integral indices. ?To make it work with a map-style
? ? ? dataset with non-integral indices/keys, a custom sampler must be provided.
? ? """

? ? def __getitem__(self, index):
? ? ? ? raise NotImplementedError

? ? def __add__(self, other):
? ? ? ? return ConcatDataset([self, other])

? ? # No `def __len__(self)` default?
? ? # See NOTE [ Lack of Default `__len__` in Python Abstract Base Classes ]
? ? # in pytorch/torch/utils/data/sampler.py

import torch
from torch.utils.data import Dataset, DataLoader, TensorDataset
from torch.autograd import Variable
import numpy as np
import pandas as pd

value_df = pd.read_csv('data1.csv'）
value_array = np.array(value_df)
print("value_array.shape =", value_array.shape) ?# (73700, 300)
value_size = value_array.shape[0] ?# 73700
train_size = int(0.7*value_size)

train_array = val_array[:train_size] ?
train_label_array = val_array[60:train_size+60]

class DealDataset(Dataset):
? ? """
? ? ? ? 下載數據、初始化數據，都可以在這里完成
? ? """

? ? def __init__(self, *arrays):
? ? ? ? assert all(arrays[0].shape[0] == array.shape[0] for array in arrays)
? ? ? ? self.arrays = arrays

? ? def __getitem__(self, index):
? ? ? ? return tuple(array[index] for array in self.arrays)

? ? def __len__(self):
? ? ? ? return self.arrays[0].shape[0]


# 實例化這個類，然后我們就得到了Dataset類型的數據，記下來就將這個類傳給DataLoader，就可以了。
train_dataset = DealDataset(train_array, train_label_array)

train_loader2 = DataLoader(dataset=train_dataset,
? ? ? ? ? ? ? ? ? ? ? ? ? ?batch_size=32,
? ? ? ? ? ? ? ? ? ? ? ? ? ?shuffle=True)

for epoch in range(2):
? ? for i, data in enumerate(train_loader2):
? ? ? ? # 將數據從 train_loader 中讀出來,一次讀取的樣本數是32個
? ? ? ? inputs, labels = data

? ? ? ? # 將這些數據轉換成Variable類型
? ? ? ? inputs, labels = Variable(inputs), Variable(labels)

? ? ? ? # 接下來就是跑模型的環(huán)節(jié)了，我們這里使用print來代替
? ? ? ? print("epoch：", epoch, "的第", i, "個inputs", inputs.data.size(), "labels", labels.data.size())

class TensorDataset(Dataset):
? ? r"""Dataset wrapping tensors.

? ? Each sample will be retrieved by indexing tensors along the first dimension.

? ? Arguments:
? ? ? ? *tensors (Tensor): tensors that have the same size of the first dimension.
? ? """

? ? def __init__(self, *tensors):
? ? ? ? assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
? ? ? ? self.tensors = tensors

? ? def __getitem__(self, index):
? ? ? ? return tuple(tensor[index] for tensor in self.tensors)

? ? def __len__(self):
? ? ? ? return self.tensors[0].size(0)

import torch
from torch.utils.data import Dataset, DataLoader, TensorDataset
from torch.autograd import Variable
import numpy as np
import pandas as pd

value_df = pd.read_csv('data1.csv'）
value_array = np.array(value_df)
print("value_array.shape =", value_array.shape) ?# (73700, 300)
value_size = value_array.shape[0] ?# 73700
train_size = int(0.7*value_size)

train_array = val_array[:train_size] ?
train_tensor = torch.tensor(train_array, dtype=torch.float32).to(device)
train_label_array = val_array[60:train_size+60]
train_labels_tensor = torch.tensor(train_label_array,dtype=torch.float32).to(device)

train_dataset = TensorDataset(train_tensor, train_labels_tensor)
train_loader = DataLoader(dataset=train_dataset,
? ? ? ? ? ? ? ? ? ? ? ? ? batch_size=100,
? ? ? ? ? ? ? ? ? ? ? ? ? shuffle=False,
? ? ? ? ? ? ? ? ? ? ? ? ? num_workers=0)

for epoch in range(2):
? ? for i, data in enumerate(train_loader):
? ? ? ? inputs, labels = data
? ? ? ? inputs, labels = Variable(inputs), Variable(labels)
? ? ? ? print(epoch, i, "inputs", inputs.data.size(), "labels", labels.data.size())