ENH simplify formatting math

This commit is contained in:
Nathan Dwarshuis 2022-07-23 23:51:54 -04:00
parent e7c05bb0d7
commit 25b2ddf2fd
1 changed files with 9 additions and 11 deletions

View File

@ -2,6 +2,7 @@ local M = {}
local __tostring = tostring local __tostring = tostring
local __math_floor = math.floor local __math_floor = math.floor
local __math_log = math.log
local __math_ceil = math.ceil local __math_ceil = math.ceil
local __string_format = string.format local __string_format = string.format
@ -23,21 +24,18 @@ M.round_to_string = function(x, places)
end end
end end
-- ASSUME domain is (0, inf)
M.precision_round_to_string = function(x, sig_fig) M.precision_round_to_string = function(x, sig_fig)
sig_fig = sig_fig or 4 local adj = x == 0 and 1 or __math_floor(__math_log(x, 10)) + 1
if x < 10 then return M.round_to_string(x, sig_fig - 1) return M.round_to_string(x, (sig_fig or 4) - adj)
elseif x < 100 then return M.round_to_string(x, sig_fig - 2)
elseif x < 1000 then return M.round_to_string(x, sig_fig - 3)
else return M.round_to_string(x, sig_fig - 4)
end
end end
local PREFIXES = {'', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi'}
-- ASSUME domain is (0, inf)
M.convert_data_val = function(x) M.convert_data_val = function(x)
if x < 1024 then return '', x local z = x == 0 and 0 or __math_floor(__math_log(x, 2) / 10)
elseif x < 1048576 then return 'Ki', x / 1024 return PREFIXES[z + 1], x / 2 ^ (10 * z)
elseif x < 1073741824 then return 'Mi', x / 1048576
else return 'Gi', x / 1073741824
end
end end
return M return M