popularity.py

""" plot popularity Zipf curve 

usage: 
1. run traceAnalyzer: `./traceAnalyzer /path/trace trace_format --common`, 
this will generate some output, including popularity result, trace.popularity
2. plot popularity using this script: 
`python3 popularity.py trace.popularity`

"""

import os, sys
import numpy as np
import matplotlib.pyplot as plt
from collections import Counter
from typing import List, Dict, Tuple
import logging

sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from utils.trace_utils import extract_dataname
from utils.plot_utils import FIG_DIR, FIG_TYPE


logger = logging.getLogger("popularity")


def load_popularity_data(datapath):
    """load popularity plot data from C++ computation"""

    ifile = open(datapath)
    data_line = ifile.readline()
    desc_line = ifile.readline()
    assert "# freq (sorted):cnt" in desc_line, (
        "the input file might not be popularity freq data file " + "data " + datapath
    )

    sorted_freq = []
    freq_cnt = Counter()
    for line in ifile:
        freq, cnt = [int(i) for i in line.strip("\n").split(":")]
        for i in range(cnt):
            sorted_freq.append(freq)
        freq_cnt[freq] += 1

    ifile.close()

    return sorted_freq, freq_cnt


def plot_popularity_Zipf(datapath, figname_prefix=""):
    from scipy import stats
    from scipy.optimize import minimize
    from scipy.optimize import curve_fit

    if len(figname_prefix) == 0:
        figname_prefix = extract_dataname(datapath)

    sorted_freq, _ = load_popularity_data(datapath)

    plt.plot(
        sorted_freq,
    )
    plt.xlabel("Object rank")
    plt.ylabel("Frequency")
    plt.grid(linestyle="--")
    plt.xscale("log")
    plt.yscale("log")
    plt.savefig(
        "{}/{}_pop_rank.{}".format(FIG_DIR, figname_prefix, FIG_TYPE),
        bbox_inches="tight",
    )
    plt.clf()
    logger.info(
        "save fig to {}/{}_pop_rank.{}".format(FIG_DIR, figname_prefix, FIG_TYPE)
    )

    x = np.log(np.arange(1, 1 + len(sorted_freq)))
    y = np.log(np.array(sorted_freq))
    slope, intercept, r_value, p_value, std_err = stats.linregress(x, y)

    if sorted_freq[0] < 100:
        s = "{:48} {:12} obj alpha 0, r^2 0 (the most popular object has less than 100 requests)".format(
            figname_prefix,
            len(sorted_freq),
        )
    else:
        s = "{:48} {:12} obj alpha {:.4f}, r^2 {:.4f}".format(
            figname_prefix, len(sorted_freq), -slope, r_value * r_value
        )

    logger.info(s)

    return s


if __name__ == "__main__":
    import argparse

    ap = argparse.ArgumentParser()
    ap.add_argument(
        "datapath",
        type=str,
        help="data path, should have the format of dataname.popularity",
    )
    ap.add_argument(
        "--figname-prefix", type=str, default="", help="the prefix of figname"
    )
    p = ap.parse_args()

    plot_popularity_Zipf(p.datapath, p.figname_prefix)