video_try_on/extract_dino_fea_vtt.py at master · scnuhealthy/video_try_on · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
import torch
import torch.utils.data as data
from PIL import Image
import torchvision.transforms as T
from sklearn.decomposition import PCA
from PIL import Image
import numpy as np
import os
import glob

class ClothDataSet(data.Dataset):
    def __init__(self, root):
        files = os.listdir(root)
        self.root = root
        self.files = []
        for root, dirs, files in os.walk(root):
            for f in files:
                # if 'cloth_front' in f and 'mask' not in f:
                self.files.append(os.path.join(root,f))
        self.transform = T.Compose([
            T.Resize((280,224), interpolation=T.InterpolationMode.BICUBIC),
            T.ToTensor(),
            T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
        ])
    def __getitem__(self,index):
        cloth_file = self.files[index]
        img = Image.open(cloth_file).convert('RGB')
        img = self.transform(img)
        return cloth_file, img

    def __len__(self):
        return len(self.files)

class CPDataLoader(object):
    def __init__(self, batch_size, workers, shuffle, dataset, collate_fn=None):
        super(CPDataLoader, self).__init__()

        if shuffle :
            train_sampler = torch.utils.data.sampler.RandomSampler(dataset)
        else:
            train_sampler = torch.utils.data.SequentialSampler(dataset)

        self.data_loader = torch.utils.data.DataLoader(
                dataset, batch_size=batch_size, shuffle=False,
                num_workers=workers, pin_memory=True, drop_last=False, sampler=train_sampler,collate_fn=collate_fn)
        self.dataset = dataset
        self.data_iter = self.data_loader.__iter__()

    def next_batch(self):
        try:
            batch = self.data_iter.__next__()
        except StopIteration:
            self.data_iter = self.data_loader.__iter__()
            batch = self.data_iter.__next__()

        return batch

def visualize(features, h, w):
    # vis code from https://github.com/dichotomies/N3F/tree/master/feature_extractor
    dim = features.shape[-1]
    features = features.reshape(-1, h, w, dim).permute(0, 3, 1, 2)
    all_features = features.cpu()
    pca = PCA(n_components=3)
    N, C, H, W = all_features.shape
    all_features = all_features.permute(0, 2, 3, 1).view(-1, C).numpy()
    pca_features = pca.fit_transform(all_features)
    pca_features = (pca_features - pca_features.min()) / (pca_features.max() - pca_features.min())
    pca_features = pca_features * 255
    pca_features = pca_features.reshape(h, w, 3)
    vis_img = Image.fromarray(pca_features.astype(np.uint8))
    vis_img.save('vis.jpg')


if  __name__ == '__main__':

    d = ClothDataSet('/data1/hzj/CUHK_dataset/lingteng_dance/cloth/')
    feature_save_root = '/data1/hzj/CUHK_dataset/lingteng_dance/dino_fea/'
    print(len(d))
    # d = ClothDataSet('/data1/hzj/192_256/clothes_person/img')
    # feature_save_root = '/data1/hzj/192_256/clothes_person/dino_fea'

    model = torch.hub.load('facebookresearch/dinov2', 'dinov2_vitl14')
    model = model.cuda()

    # d = ClothDataSet('/data1/hzj/zalando-hd-resized/test/cloth/')
    # feature_save_root = '/data1/hzj/zalando-hd-resized/test/dino_fea/'
    # d = ClothDataSet('/data1/hzj/DressCode/upper_body/images')
    # feature_save_root = '/data1/hzj/DressCode/upper_body/dino_fea'
    dataloader = CPDataLoader(1,1,False,d)

    for i, batch in enumerate(dataloader.data_loader):
        names = batch[0]
        imgs_tenosr = batch[1]
        with torch.no_grad():
            features_dict = model.forward_features(imgs_tenosr.cuda())
        features_patchtokens = features_dict['x_norm_patchtokens']
        feature_clstoken = features_dict['x_norm_clstoken'].unsqueeze(1)
        features = torch.cat([feature_clstoken,features_patchtokens],1)
        ## save tensor
        print(names)
        for j in range(len(names)):
            save_feature = features[j]
            save_name = names[j].split('/')[-1][:-4] + '.pt'
            save_path = os.path.join(feature_save_root, save_name)
            torch.save(save_feature, save_path)
        ## visualize
        # h, w = int(imgs_tenosr.shape[2] / model.patch_embed.patch_size[0]), int(
        #     imgs_tenosr.shape[3] / model.patch_embed.patch_size[1]
        # )
        # visualize(features_patchtokens[1], h, w)

    # img = Image.open('/data1/hzj/zalando-hd-resized/train/cloth/00024_00.jpg')
    # transform = T.Compose([
    #     # T.Resize(256, interpolation=T.InterpolationMode.BICUBIC),
    #     # T.CenterCrop(224),
    #     # T.Resize((574,448), interpolation=T.InterpolationMode.BICUBIC),
    #     T.Resize((280,224), interpolation=T.InterpolationMode.BICUBIC),
    #     T.ToTensor(),
    #     T.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
    # ])

    # img = transform(img)[:3].unsqueeze(0)
    # img = img.cuda()

    # h, w = int(img.shape[2] / model.patch_embed.patch_size[0]), int(
    #     img.shape[3] / model.patch_embed.patch_size[1]
    # )
    # with torch.no_grad():
    #     # 将图像张量传递给dinov2_vits14模型获取特征
    #     features_dict = model.forward_features(img)
    #     features_patchtokens = features_dict['x_norm_patchtokens']
    #     feature_clstoken = features_dict['x_norm_clstoken'].unsqueeze(1)
    #     print(feature_clstoken.shape,features_patchtokens.shape)
    #     features = torch.cat([feature_clstoken,features_patchtokens],1)
    #     print(features.shape)

    # visualize(features_patchtokens, h, w)