rFID of VAR is very large

[sunset-clouds]

I’ve noticed that many people say the rFID tested with the checkpoint provided by VAR is 0.92. However, when I followed the tutorial provided by VAR, the rFID I got was 2.70. I’m not sure what the reason is.

Firstly, we load imagenet val dataset, and resize by center_crop_arr (provided in LLamaGEN). We then obtain the reconstruction image, and saved in PNG format by Image.fromarray(sample).save(f"{sample_folder_dir}/{index:06d}.png"). Finally, we process saved reconstrution images to a npz file by create_npz_from_sample_folder.

import os import os.path as osp import torch, torchvision import random import numpy as np import PIL.Image as PImage, PIL.ImageDraw as PImageDraw import torch torch.backends.cuda.matmul.allow_tf32 = True torch.backends.cudnn.allow_tf32 = True import torch.nn.functional as F import torch.distributed as dist from torch.utils.data import DataLoader from torchvision.datasets import ImageFolder from torch.utils.data.distributed import DistributedSampler from torchvision import transforms from tqdm import tqdm from PIL import Image import numpy as np import argparse import itertools

from models import VQVAE, build_vae_var from metric import PSNR, LPIPS, SSIM from augmentation import center_crop_arr

def create_npz_from_sample_folder(sample_dir, num=50000): """ Builds a single .npz file from a folder of .png samples. """ samples = [] for i in tqdm(range(num), desc="Building .npz file from samples"): sample_pil = Image.open(f"{sample_dir}/{i:06d}.png") sample_np = np.asarray(sample_pil).astype(np.uint8) samples.append(sample_np) samples = np.stack(samples) assert samples.shape == (num, samples.shape[1], samples.shape[2], 3) npz_path = f"{sample_dir}.npz" np.savez(npz_path, arr_0=samples) print(f"Saved .npz file to {npz_path} [shape={samples.shape}].") return npz_path

def load_dataset(data_path, batch_size=16): transform = transforms.Compose([ transforms.Lambda(lambda pil_image: center_crop_arr(pil_image, 256)), transforms.ToTensor(), transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True) ]) val_dataset = ImageFolder(root=os.path.join(data_path, 'val'), transform=transform) len_val_set = len(val_dataset) dataloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=6, drop_last=False) return dataloader, len_val_set

def main(): sample_folder_dir = "/projects/yuanai/processed_data/rFID/baselines/VAR" save_npz_name = "var_reconstruction_imagenet256.npz"

### load dataset
data_path = "/projects/yuanai/data/ImageNet/"
val_dataloader, len_val_set = load_dataset(data_path, batch_size=16)
num_fid_samples = 50000

###load the vae checkpoint
vae, var = build_vae_var(
    V=4096, Cvae=32, ch=160, share_quant_resi=4,    # hard-coded VQVAE hyperparameters
    device='cuda', patch_nums= (1, 2, 3, 4, 5, 6, 8, 10, 13, 16),
    num_classes=1000, depth=16, shared_aln=False,
)
vae_ckpt = "/projects/yuanai/processed_data/checkpoint/VAR/vae_ch160v4096z32.pth"
vae.load_state_dict(torch.load(vae_ckpt, map_location='cpu'), strict=True)
vae = vae.cuda()
vae.eval()

psnr_metric = PSNR()
ssim_metric = SSIM()
lpips_metric = LPIPS()
ssim, psnr, lpips = 0.0, 0.0, 0.0
total = 0
for idx, (x, _) in enumerate(val_dataloader):
    x = x.cuda()
    with torch.no_grad():
        x_rec = vae.img_to_reconstructed_img(x, v_patch_nums=(1, 2, 3, 4, 5, 6, 8, 10, 13, 16), last_one=True)
        batch_lpips = lpips_metric(x, x_rec).sum()

        samples = torch.clamp(127.5 * x_rec + 128.0, 0, 255).permute(0, 2, 3, 1).to("cpu", dtype=torch.uint8).numpy()

        # Save samples to disk as individual .png files
        for i, sample in enumerate(samples):
            index = i + total
            Image.fromarray(sample).save(f"{sample_folder_dir}/{index:06d}.png")

        total += 16

        x_norm = (x + 1.0)/2.0
        x_rec_norm = (x_rec + 1.0)/2.0

        batch_psnr = psnr_metric(x_norm, x_rec_norm).sum()
        batch_ssim = ssim_metric(x_norm, x_rec_norm).sum()

        ssim += batch_ssim.item()
        psnr += batch_psnr.item()
        lpips += batch_lpips.item()

eval_psnr = psnr/len_val_set
eval_ssim = ssim/len_val_set
eval_lpips = lpips/len_val_set
print("PSNR:"+str(eval_psnr)+"  SSIM:"+str(eval_ssim)+ "  LPIPS:"+str(eval_lpips))

create_npz_from_sample_folder(sample_folder_dir, num_fid_samples)

if name == "main": main()

Based the npz.file, we calculate rFID by Open AI toolkit by "python evaluator.py VIRTUAL_imagenet256_labeled.npz our_sampled_imagenet256.npz", then the calculated rFID is: Inception Score: 56.86065673828125 FID: 2.70789722116308 sFID: 4.6903826389984715 Precision: 0.74194 Recall: 0.6662

Can anyone tell me why there is a problem with the rFID test results?