From 2bdd26e4904f1de6db26acc14a914e0c4957353f Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Thu, 29 Oct 2015 22:44:35 +0000 Subject: [PATCH 1/4] build-pkg: add logging function echo echo is format + print log is echo('[build-pkg]', ...) --- tools/build-pkg.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/build-pkg.lua b/tools/build-pkg.lua index 63452c80..7b7497a3 100755 --- a/tools/build-pkg.lua +++ b/tools/build-pkg.lua @@ -59,8 +59,12 @@ local TARGETS = { 'x86_64-w64-mingw32.shared', } +local function echo(fmt, ...) + print(fmt:format(...)) +end + local function log(fmt, ...) - print('[build-pkg]', fmt:format(...)) + echo('[build-pkg]\t' .. fmt, ...) end -- based on http://lua-users.org/wiki/SplitJoin From 28b75d2b58bd0073c4409ed498be3a93fe04f244 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Thu, 29 Oct 2015 22:47:27 +0000 Subject: [PATCH 2/4] build-pkg: print progress Example: [ 4/100] The build is expected to complete in 0.3 hours, on Thu Oct 29 23:03:27 2015 --- tools/build-pkg.lua | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tools/build-pkg.lua b/tools/build-pkg.lua index 7b7497a3..a18ff18c 100755 --- a/tools/build-pkg.lua +++ b/tools/build-pkg.lua @@ -508,7 +508,9 @@ local function buildPackages(items, item2deps) end return false end - for _, item in ipairs(items) do + local nitems = #items + local started_at = os.time() + for i, item in ipairs(items) do if not brokenDep(item) then local files = buildItem(item, item2deps, file2item) if isBuilt(item, files) then @@ -524,6 +526,16 @@ local function buildPackages(items, item2deps) log('Item %s depends on broken item %s', item, brokenDep(item)) end + local now = os.time() + local spent = now - started_at + local predicted_duration = spent * nitems / i + local predicted_end = started_at + predicted_duration + local predicted_end_str = os.date("%c", predicted_end) + local predicted_wait = predicted_end - now + local predicted_wait_hours = predicted_wait / 3600.0 + echo("[%3d/%d] The build is expected to complete " .. + "in %0.1f hours, on %s", i, nitems, + predicted_wait_hours, predicted_end_str) end return unbroken, item2files end From fec25ef4513f17b74021035fc9a9745b2c47ea3a Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sat, 31 Oct 2015 21:39:22 +0300 Subject: [PATCH 3/4] build-pkg: separate progress printing code --- tools/build-pkg.lua | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/tools/build-pkg.lua b/tools/build-pkg.lua index a18ff18c..f3eb1474 100755 --- a/tools/build-pkg.lua +++ b/tools/build-pkg.lua @@ -494,6 +494,31 @@ local function isBuilt(item, files) return false end +local PROGRESS = "[%3d/%d] " .. + "The build is expected to complete in %0.1f hours, " .. + "on %s" +local function progressPrinter(items) + local nitems = #items + local started_at = os.time() + local done = 0 + local printer = {} + function printer:advance(i) + done = i + end + function printer:status() + local now = os.time() + local spent = now - started_at + local predicted_duration = spent * nitems / done + local predicted_end = started_at + predicted_duration + local predicted_end_str = os.date("%c", predicted_end) + local predicted_wait = predicted_end - now + local predicted_wait_hours = predicted_wait / 3600.0 + return PROGRESS:format(done, nitems, + predicted_wait_hours, predicted_end_str) + end + return printer +end + -- build all packages, save filelist to list file local function buildPackages(items, item2deps) local broken = {} @@ -508,8 +533,7 @@ local function buildPackages(items, item2deps) end return false end - local nitems = #items - local started_at = os.time() + local progress_printer = progressPrinter(items) for i, item in ipairs(items) do if not brokenDep(item) then local files = buildItem(item, item2deps, file2item) @@ -526,16 +550,8 @@ local function buildPackages(items, item2deps) log('Item %s depends on broken item %s', item, brokenDep(item)) end - local now = os.time() - local spent = now - started_at - local predicted_duration = spent * nitems / i - local predicted_end = started_at + predicted_duration - local predicted_end_str = os.date("%c", predicted_end) - local predicted_wait = predicted_end - now - local predicted_wait_hours = predicted_wait / 3600.0 - echo("[%3d/%d] The build is expected to complete " .. - "in %0.1f hours, on %s", i, nitems, - predicted_wait_hours, predicted_end_str) + progress_printer:advance(i) + echo(progress_printer:status()) end return unbroken, item2files end From 8708cf1ed3c47fca1db46b5a91b008a8f9556c69 Mon Sep 17 00:00:00 2001 From: Boris Nagaev Date: Sat, 31 Oct 2015 23:00:42 +0300 Subject: [PATCH 4/4] build-pkg: improve the progress printer accuracy There are 20 huge packages like gcc that are compiled for a half of the build time. The time left expected by the progess printer changes dramatically after building a huge package. Information about huge packages smoothes the changes. --- tools/build-pkg.lua | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/tools/build-pkg.lua b/tools/build-pkg.lua index f3eb1474..d39dec11 100755 --- a/tools/build-pkg.lua +++ b/tools/build-pkg.lua @@ -494,26 +494,55 @@ local function isBuilt(item, files) return false end +-- script building HUGE_TIMES from MXE main log +-- https://gist.github.com/starius/3ea9d953b0c30df88aa7 +local HUGE_TIMES = { + [7] = {"ocaml-native", "ffmpeg", "boost"}, + [9] = {"openssl", "qtdeclarative", "ossim", "wxwidgets"}, + [12] = {"ocaml-core", "itk", "wt"}, + [19] = {"gcc", "qtbase", "llvm"}, + [24] = {"vtk", "vtk6", "openscenegraph"}, + [36] = {"openblas", "pcl", "oce"}, + [51] = {"qt"}, +} + local PROGRESS = "[%3d/%d] " .. "The build is expected to complete in %0.1f hours, " .. "on %s" + local function progressPrinter(items) - local nitems = #items + local pkg2time = {} + for time, pkgs in pairs(HUGE_TIMES) do + for _, pkg in ipairs(pkgs) do + pkg2time[pkg] = time + end + end + -- local started_at = os.time() - local done = 0 + local sums = {} + for i, item in ipairs(items) do + local target, pkg = parseItem(item) + local expected_time = pkg2time[pkg] or 1 + sums[i] = (sums[i - 1] or 0) + expected_time + end + local total_time = sums[#sums] + local time_done = 0 + local pkgs_done = 0 local printer = {} + -- function printer:advance(i) - done = i + pkgs_done = i + time_done = sums[i] end function printer:status() local now = os.time() local spent = now - started_at - local predicted_duration = spent * nitems / done + local predicted_duration = spent * total_time / time_done local predicted_end = started_at + predicted_duration local predicted_end_str = os.date("%c", predicted_end) local predicted_wait = predicted_end - now local predicted_wait_hours = predicted_wait / 3600.0 - return PROGRESS:format(done, nitems, + return PROGRESS:format(pkgs_done, #items, predicted_wait_hours, predicted_end_str) end return printer