#!/usr/bin/python import csv,sys,os,shutil,ast,json,hashlib,pprint,math refseqcsv='reference-sequence.csv' anchorcsv='streams.csv' referenceSequences = '../ReferenceSequences/' metricsDirectory = 'Metrics/' configDirectory = 'CFG/' #scenarios = [ f.path for f in os.scandir("./") if f.is_dir() ] scenarios = [ 'Scenario-5-Gaming'] # for all scenarios for s in scenarios: # testfolder = [ ff.path for ff in os.scandir(s) if ff.is_dir() ] testfolder = [ 'Scenario-5-Gaming/AV1' ] rfile = s + '/' + refseqcsv # does reference sequence exists if os.path.isfile(rfile): # for test models for test in testfolder: adir = test + '/' adir = adir.replace('.\\','') adir = adir.replace('\\','/') afile = adir + anchorcsv # does anchor file exist if os.path.isfile(afile): # open reference sequence and anchor with open(rfile) as csv_reference, open(afile) as csv_anchor: reference = csv.reader(csv_reference, delimiter=',') reference_header = next(reference) reference_rows = [] for row in reference: reference_rows.append(row) # get anchors anchor_rows = [] anchor = csv.reader(csv_anchor, delimiter=',') anchor_header = next(anchor) for row in anchor: anchor_rows.append(row) # For each anchor, get the metrics for i in range(anchor.line_num-1): # determine codec print(anchor_rows[i]) codec = anchor_rows[i][4][3:5] # open the metrics file file = adir + '/' + metricsDirectory + anchor_rows[i][4] + '.csv' # create alternative file if not os.path.isfile(file): file = file.replace("03", "01") file = file.replace("04", "02") if os.path.isfile(file): with open(file) as csv_metrics: metrics = csv.reader(csv_metrics, delimiter=',') metrics_header = next(metrics) metrics_rows = [] for row in metrics: metrics_rows.append(row) # find correct reference sequence for j in range(reference.line_num-1): if anchor_rows[i][2] == reference_rows[j][0]: sequence = reference_rows[j][6] # search sequence in metrics for k in range(metrics.line_num-1): # check if the first column of the metric file is either the key or the sequence name if sequence in metrics_rows[k][0] or reference_rows[j][0] == metrics_rows[k][0]: metricsoutfile = adir + '/' + metricsDirectory + anchor_rows[i][0] + '.csv' with open(metricsoutfile, mode='w',newline='') as outfile: out_writer = csv.writer(outfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) if reference_rows[j][5] == "1": # HDR Metrics out_writer.writerow(['parameter','bitrate','y_psnr','u_psnr','v_psnr','psnr','y_wpsnr','u_wpsnr','v_wpsnr','wpsnr','psnr100','de100','bitrate_file','encode_time','decode_time','recon_md5','bitstream_md5']) values = ast.literal_eval(anchor_rows[i][5]) for q in values: if k < metrics.line_num-1 and metrics_rows[k][1] == str(q): # If it is AV1, obu ending, otherwise bin if codec == 'AV': anchorfile = test + '/' + anchor_rows[i][0] + '/' + anchor_rows[i][0] + '-' + repr(q) + '.obu' else: anchorfile = test + '/' + anchor_rows[i][0] + '/' + anchor_rows[i][0] + '-' + repr(q) + '.bin' anchorfile_bis = test + '/' + anchor_rows[i][0] + '/' + anchor_rows[i][0] + '-' + repr(q) + '.bis' if not os.path.isfile(anchorfile_bis): anchorfile_bis = anchorfile # if file exists, get bitrate and md5 if os.path.isfile(anchorfile): bitrate_file = os.path.getsize(anchorfile)*8/(float(reference_rows[j][4])*1000) md5_hash = hashlib.md5() with open(anchorfile,"rb") as f: # Read and update hash in chunks of 4K for byte_block in iter(lambda: f.read(4096),b""): md5_hash.update(byte_block) md5 = md5_hash.hexdigest() else: bitrate_file = 0; md5 = "unknown"; # Assign parameters if codec == 'HM': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = float(metrics_rows[k][6]) y_wpsnr = float(metrics_rows[k][7]) u_wpsnr = float(metrics_rows[k][8]) v_wpsnr = float(metrics_rows[k][9]) wpsnr = float(metrics_rows[k][10]) de100 = float(metrics_rows[k][12]) psnrl100 = float(metrics_rows[k][11]) bitrate_log = float(metrics_rows[k][13]) encode_time = float(metrics_rows[k][14]) decode_time = float(metrics_rows[k][15]) recon_md5 = "unknown" bitstream_md5 = md5 elif codec == 'AV': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = (6*y_psnr + u_psnr + v_psnr)/8 y_wpsnr = float(metrics_rows[k][6]) u_wpsnr = float(metrics_rows[k][7]) v_wpsnr = float(metrics_rows[k][8]) wpsnr = (6*y_wpsnr + u_wpsnr + v_wpsnr)/8 de100 = float(metrics_rows[k][9]) psnrl100 = float(metrics_rows[k][10]) bitrate_log = bitrate encode_time = float(metrics_rows[k][11]) decode_time = float(metrics_rows[k][12]) recon_md5 = "unknown" bitstream_md5 = md5 elif codec == 'ET': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = (6*y_psnr + u_psnr + v_psnr)/8 y_wpsnr = float(metrics_rows[k][6]) u_wpsnr = float(metrics_rows[k][7]) v_wpsnr = float(metrics_rows[k][8]) wpsnr = (6*y_wpsnr + u_wpsnr + v_wpsnr)/8 de100 = float(metrics_rows[k][9]) psnrl100 = float(metrics_rows[k][10]) bitrate_log = bitrate encode_time = float(metrics_rows[k][11]) decode_time = float(metrics_rows[k][12]) recon_md5 = metrics_rows[k][13] bitstream_md5 = metrics_rows[k][14] elif codec == 'VT': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = float(metrics_rows[k][6]) y_wpsnr = float(metrics_rows[k][7]) u_wpsnr = float(metrics_rows[k][8]) v_wpsnr = float(metrics_rows[k][9]) wpsnr = (6*y_wpsnr + u_wpsnr + v_wpsnr)/8 de100 = float(metrics_rows[k][11]) psnrl100 = float(metrics_rows[k][10]) bitrate_log = bitrate encode_time = float(metrics_rows[k][12]) decode_time = float(metrics_rows[k][13]) recon_md5 = metrics_rows[k][14] bitstream_md5 = metrics_rows[k][15] else: print("Warning codec unknown", codec) exit if( abs(bitrate - bitrate_file) > 10 ): print("Warning: Bitrates do not match: reported: ", bitrate, "file: ", bitrate_file, "for: ", anchorfile) if( abs(psnr - (6*y_psnr + u_psnr + v_psnr)/8) > 0.1 ): print("Warning: PSNR does not match", psnr, (6*y_psnr + u_psnr + v_psnr)/8, anchorfile) if( abs(wpsnr - (6*y_wpsnr + u_wpsnr + v_wpsnr)/8) > 0.1 ): print("Warning: PSNR does not match", psnr, (6*y_psnr + u_psnr + v_psnr)/8, anchorfile) if( int(metrics_rows[k][1]) != int(q) ): print("Warning: Parameter does not match", metrics_rows[k][1], q, anchorfile) if ( md5 != bitstream_md5 ): print("Warning: Bitstream md5 does not match", md5, bitstream_md5, anchorfile) bitrate_log = bitrate_file out_writer.writerow([parameter, '{:.3f}'.format(round(bitrate,3)), '{:.3f}'.format(round(y_psnr,3)), '{:.3f}'.format(round(u_psnr,3)), '{:.3f}'.format(round(v_psnr,3)), '{:.3f}'.format(round(psnr,3)), '{:.3f}'.format(round(y_wpsnr,3)), '{:.3f}'.format(round(u_wpsnr,3)), '{:.3f}'.format(round(v_wpsnr,3)), '{:.3f}'.format(round(wpsnr,3)), '{:.3f}'.format(round(psnrl100,3)), '{:.3f}'.format(round(de100,3)), '{:.3f}'.format(round(bitrate_log,3)), '{:.3f}'.format(round(encode_time,3)), '{:.3f}'.format(round(decode_time,3)), recon_md5, bitstream_md5]) k = k + 1 else: # SDR Metrics out_writer.writerow(['parameter','bitrate','y_psnr','u_psnr','v_psnr','psnr','ms_ssim','vmaf','bitrate_log','encode_time','decode_time','recon_md5','bitstream_md5']) # print("write header", metricsoutfile) # for every q parameter values = ast.literal_eval(anchor_rows[i][5]) for q in values: if k < metrics.line_num-1 and metrics_rows[k][1] == str(q): # If it is AV1, obu ending, otherwise bin if codec == 'AV': anchorfile = test + '/' + anchor_rows[i][0] + '/' + anchor_rows[i][0] + '-' + repr(q) + '.obu' anchorfile_bis = test + '/' + anchor_rows[i][0] + '/' + anchor_rows[i][0] + '-' + repr(q) + '.bis' if not os.path.isfile(anchorfile_bis): anchorfile_bis = anchorfile if not os.path.isfile(anchorfile): anchorfile = test + '/' + anchor_rows[i-4][0] + '/' + anchor_rows[i-4][0] + '-' + repr(q) + '.obu' print("Alternate Anchor", anchor_rows[i][0], anchorfile) else: anchorfile = test + '/' + anchor_rows[i][0] + '/' + anchor_rows[i][0] + '-' + repr(q) + '.bin' anchorfile_bis = test + '/' + anchor_rows[i][0] + '/' + anchor_rows[i][0] + '-' + repr(q) + '.bis' if not os.path.isfile(anchorfile_bis): anchorfile_bis = anchorfile if not os.path.isfile(anchorfile): if codec == 'HM' or codec == 'SC': anchorfile = test + '/' + anchor_rows[i-4][0] + '/' + anchor_rows[i-4][0] + '-' + repr(q) + '.bin' anchorfile_bis = test + '/' + anchor_rows[i-4][0] + '/' + anchor_rows[i-4][0] + '-' + repr(q) + '.bis' if os.path.isfile(anchorfile_bis): print("Using BIS file", anchor_rows[i][0], anchorfile_bis) else: anchorfile_bis = anchorfile elif anchor_rows[i][0] == "S3-T35-ETM": anchorfile = test + '/' + anchor_rows[i-18][0] + '/' + anchor_rows[i-18][0] + '-' + repr(q) + '.bin' anchorfile_bis = test + '/' + anchor_rows[i-18][0] + '/' + anchor_rows[i-18][0] + '-' + repr(q) + '.bis' if os.path.isfile(anchorfile_bis): print("Using BIS file", anchor_rows[i][0], anchorfile_bis) else: anchorfile_bis = anchorfile else: anchorfile = test + '/' + anchor_rows[i-2][0] + '/' + anchor_rows[i-2][0] + '-' + repr(q) + '.bin' anchorfile_bis = test + '/' + anchor_rows[i-2][0] + '/' + anchor_rows[i-2][0] + '-' + repr(q) + '.bis' if os.path.isfile(anchorfile_bis): print("Using BIS file", anchor_rows[i][0], anchorfile_bis) else: anchorfile_bis = anchorfile print("Alternate Anchor", anchor_rows[i][0], anchorfile) # if file exists, get bitrate and md5 if os.path.isfile(anchorfile): bitrate_file = os.path.getsize(anchorfile)*8/(float(reference_rows[j][4])*1000) bitrate_efs = os.path.getsize(anchorfile_bis)*8/(float(reference_rows[j][4])*1000) md5_hash = hashlib.md5() with open(anchorfile,"rb") as f: # Read and update hash in chunks of 4K for byte_block in iter(lambda: f.read(4096),b""): md5_hash.update(byte_block) md5 = md5_hash.hexdigest() else: bitrate_file = 0 bitrate_efs = 0 md5 = "unknown" bitrate_log = bitrate_file # Assign parameters if codec == 'JM': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) psnr = float(metrics_rows[k][3]) y_psnr = float(metrics_rows[k][4]) u_psnr = float(metrics_rows[k][5]) v_psnr = float(metrics_rows[k][6]) # psnr = (6*y_psnr + u_psnr + v_psnr)/8 if(float(metrics_rows[k][7]) < 1.0): ms_ssim = float(-10.0*math.log(1-float(metrics_rows[k][7]), 10)) else: ms_ssim = float(metrics_rows[k][7]) vmaf = float(metrics_rows[k][8]) encode_time = 0. # float(metrics_rows[k][6]) decode_time = 0. # float(metrics_rows[k][7]) recon_md5 = metrics_rows[k][10] bitstream_md5 = md5 elif codec == 'HM' or codec == 'SC': parameter = int(metrics_rows[k][1]) if "3" in s or "5" in s: bitrate = bitrate_efs bitrate_log = float(metrics_rows[k][2]) if( bitrate_log != 0 and abs((bitrate - bitrate_log)/bitrate_log) > 0.2 ): print("Warning: Significant bitrate differences: reported: ", bitrate_log, "file: ", bitrate_file, "for: ", anchorfile, codec) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = (6*y_psnr + u_psnr + v_psnr)/8 if(float(metrics_rows[k][9]) < 1.0): ms_ssim = float(-10.0*math.log(1-float(metrics_rows[k][9]), 10)) else: ms_ssim = float(metrics_rows[k][9]) ms_ssim = 0.; vmaf = 0.; # float(metrics_rows[k][8]) encode_time = float(metrics_rows[k][6]) decode_time = float(metrics_rows[k][7]) else: bitrate = bitrate_efs bitrate_log = float(metrics_rows[k][2]) bitrate = float(metrics_rows[k][2]) psnr = float(metrics_rows[k][3]) y_psnr = float(metrics_rows[k][4]) u_psnr = float(metrics_rows[k][5]) v_psnr = float(metrics_rows[k][6]) # psnr = (6*y_psnr + u_psnr + v_psnr)/8 if(float(metrics_rows[k][7]) < 1.0): ms_ssim = float(-10.0*math.log(1-float(metrics_rows[k][7]), 10)) else: ms_ssim = float(metrics_rows[k][7]) vmaf = float(metrics_rows[k][8]) encode_time = float(metrics_rows[k][10]) decode_time = float(metrics_rows[k][11]) recon_md5 = "unknown" bitstream_md5 = md5 elif codec == 'AV': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = (6*y_psnr + u_psnr + v_psnr)/8 if(float(metrics_rows[k][9]) < 1.0): ms_ssim = float(-10.0*math.log(1-float(metrics_rows[k][9]), 10)) else: ms_ssim = float(metrics_rows[k][9]) vmaf = float(metrics_rows[k][8]) encode_time = float(metrics_rows[k][6]) decode_time = float(metrics_rows[k][7]) recon_md5 = "unknown" bitstream_md5 = md5 elif codec == 'ET': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = (6*y_psnr + u_psnr + v_psnr)/8 if(float(metrics_rows[k][9]) < 1.0): ms_ssim = float(-10.0*math.log(1-float(metrics_rows[k][9]), 10)) else: ms_ssim = float(metrics_rows[k][9]) vmaf = float(metrics_rows[k][8]) encode_time = 0. # float(metrics_rows[k][6]) decode_time = 0. # float(metrics_rows[k][7]) recon_md5 = metrics_rows[k][10] bitstream_md5 = metrics_rows[k][11] elif codec == 'VT': parameter = int(metrics_rows[k][1]) bitrate = float(metrics_rows[k][2]) y_psnr = float(metrics_rows[k][3]) u_psnr = float(metrics_rows[k][4]) v_psnr = float(metrics_rows[k][5]) psnr = float(metrics_rows[k][6]) if(float(metrics_rows[k][7]) < 1.0): ms_ssim = float(-10.0*math.log(1-float(metrics_rows[k][7]), 10)) else: ms_ssim = float(metrics_rows[k][7]) vmaf = float(metrics_rows[k][8]) if "3" in s or "5" in s: ms_ssim = 0.; vmaf = 0.; encode_time = float(metrics_rows[k][9]) decode_time = float(metrics_rows[k][10]) recon_md5 = metrics_rows[k][11] bitstream_md5 = metrics_rows[k][12] else: print("Warning codec unknown", codec) exit if( bitrate != 0 and abs((bitrate - bitrate_log)/bitrate) > 0.005 ): print("Warning: Bitrates do not match: reported: ", bitrate_log, "file: ", bitrate, "for: ", anchorfile) if( abs(psnr - (6*y_psnr + u_psnr + v_psnr)/8) > 0.1 ): print("Warning: PSNR does not match", psnr, (6*y_psnr + u_psnr + v_psnr)/8, anchorfile) if( int(metrics_rows[k][1]) != int(q) ): print("Warning: Parameter does not match", metrics_rows[k][1], q, anchorfile) if ( md5 != bitstream_md5 ): print("Warning: Bitstream md5 does not match", md5, bitstream_md5, anchorfile) # print(parameter) out_writer.writerow([parameter, '{:.3f}'.format(round(bitrate,3)), '{:.3f}'.format(round(y_psnr,3)), '{:.3f}'.format(round(u_psnr,3)), '{:.3f}'.format(round(v_psnr,3)), '{:.3f}'.format(round(psnr,3)), '{:.3f}'.format(round(ms_ssim,3)), '{:.3f}'.format(round(vmaf,3)), '{:.3f}'.format(round(bitrate_log,3)), '{:.3f}'.format(round(encode_time,3)), '{:.3f}'.format(round(decode_time,3)), recon_md5, bitstream_md5]) k = k + 1 else: print('Metrics file', file, 'does not exist') else: print('Anchor files', afile, 'does not exist') else: print('Reference Sequences', rfile, 'does not exist')