mirror of https://git.wownero.com/dsc/mxe.git
Merge pull request #1243 from LuaAndC/build-pkg-second-pass
build-pkg: build a package second time in tree with all packages
This commit is contained in:
commit
ab6b7d9f05
|
@ -19,6 +19,9 @@ To prevent build-pkg from creating deb packages,
|
||||||
set environment variable MXE_NO_DEBS to 1
|
set environment variable MXE_NO_DEBS to 1
|
||||||
In this case fakeroot and dpkg-deb are not needed.
|
In this case fakeroot and dpkg-deb are not needed.
|
||||||
|
|
||||||
|
To switch off the second pass, set MXE_NO_SECOND_PASS to 1.
|
||||||
|
See https://github.com/mxe/mxe/issues/1111
|
||||||
|
|
||||||
To limit number of packages being built to x,
|
To limit number of packages being built to x,
|
||||||
set environment variable MXE_MAX_ITEMS to x,
|
set environment variable MXE_MAX_ITEMS to x,
|
||||||
|
|
||||||
|
@ -32,6 +35,7 @@ How to remove them: http://stackoverflow.com/a/4262545
|
||||||
|
|
||||||
local max_items = tonumber(os.getenv('MXE_MAX_ITEMS'))
|
local max_items = tonumber(os.getenv('MXE_MAX_ITEMS'))
|
||||||
local no_debs = os.getenv('MXE_NO_DEBS')
|
local no_debs = os.getenv('MXE_NO_DEBS')
|
||||||
|
local no_second_pass = os.getenv('MXE_NO_SECOND_PASS')
|
||||||
|
|
||||||
local TODAY = os.date("%Y%m%d")
|
local TODAY = os.date("%Y%m%d")
|
||||||
|
|
||||||
|
@ -174,6 +178,10 @@ local function fileExists(name)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function isSymlink(name)
|
||||||
|
return shell(("ls -l %q"):format(name)):sub(1, 1) == "l"
|
||||||
|
end
|
||||||
|
|
||||||
local function writeFile(filename, data)
|
local function writeFile(filename, data)
|
||||||
local file = io.open(filename, 'w')
|
local file = io.open(filename, 'w')
|
||||||
file:write(data)
|
file:write(data)
|
||||||
|
@ -309,6 +317,18 @@ local function sortForBuild(items, item2deps)
|
||||||
return build_list
|
return build_list
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function isDependency(item, dependency, item2deps)
|
||||||
|
for _, dep in ipairs(item2deps[item]) do
|
||||||
|
if dep == dependency then
|
||||||
|
return true
|
||||||
|
end
|
||||||
|
if isDependency(dep, dependency, item2deps) then
|
||||||
|
return true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return false
|
||||||
|
end
|
||||||
|
|
||||||
local function makeItem2Index(build_list)
|
local function makeItem2Index(build_list)
|
||||||
local item2index = {}
|
local item2index = {}
|
||||||
for index, item in ipairs(build_list) do
|
for index, item in ipairs(build_list) do
|
||||||
|
@ -350,10 +370,10 @@ local function isBlacklisted(file)
|
||||||
end
|
end
|
||||||
|
|
||||||
local GIT_INITIAL = 'initial'
|
local GIT_INITIAL = 'initial'
|
||||||
local GIT_ALL = 'first-all'
|
local GIT_ALL_PSEUDOITEM = 'all'
|
||||||
|
|
||||||
local function itemToBranch(item)
|
local function itemToBranch(item, pass)
|
||||||
return 'first-' .. item:gsub('~', '_')
|
return pass .. '-' .. item:gsub('~', '_')
|
||||||
end
|
end
|
||||||
|
|
||||||
-- creates git repo in ./usr
|
-- creates git repo in ./usr
|
||||||
|
@ -383,10 +403,10 @@ local function gitCommit(message)
|
||||||
assert(execute(cmd:format(message)))
|
assert(execute(cmd:format(message)))
|
||||||
end
|
end
|
||||||
|
|
||||||
local function gitCheckout(new_branch, deps, item2index)
|
local function gitCheckout(new_branch, deps, item2index, pass_of_deps)
|
||||||
local main_dep = deps[1]
|
local main_dep = deps[1]
|
||||||
if main_dep then
|
if main_dep then
|
||||||
main_dep = itemToBranch(main_dep)
|
main_dep = itemToBranch(main_dep, pass_of_deps)
|
||||||
else
|
else
|
||||||
main_dep = GIT_INITIAL
|
main_dep = GIT_INITIAL
|
||||||
end
|
end
|
||||||
|
@ -398,7 +418,7 @@ local function gitCheckout(new_branch, deps, item2index)
|
||||||
local cmd2 = '%s %s merge -q %s -m %q'
|
local cmd2 = '%s %s merge -q %s -m %q'
|
||||||
if not execute(cmd2:format(GIT,
|
if not execute(cmd2:format(GIT,
|
||||||
GIT_USER,
|
GIT_USER,
|
||||||
itemToBranch(deps[i]),
|
itemToBranch(deps[i], pass_of_deps),
|
||||||
message))
|
message))
|
||||||
then
|
then
|
||||||
-- probably merge conflict
|
-- probably merge conflict
|
||||||
|
@ -426,18 +446,19 @@ local function gitCheckout(new_branch, deps, item2index)
|
||||||
end
|
end
|
||||||
|
|
||||||
local function gitAdd()
|
local function gitAdd()
|
||||||
os.execute(GIT .. 'add .')
|
os.execute(GIT .. 'add --all .')
|
||||||
end
|
end
|
||||||
|
|
||||||
-- return two lists of filepaths under ./usr/
|
-- return two lists of filepaths under ./usr/
|
||||||
-- 1. new files
|
-- 1. new files
|
||||||
-- 2. changed files
|
-- 2. changed files
|
||||||
local function gitStatus()
|
local function gitStatus(item, item2deps, file2item)
|
||||||
local new_files = {}
|
local new_files = {}
|
||||||
local changed_files = {}
|
local changed_files = {}
|
||||||
local git_st = io.popen(GIT .. 'status --porcelain', 'r')
|
local git_st = io.popen(GIT .. 'status --porcelain', 'r')
|
||||||
for line in git_st:lines() do
|
for line in git_st:lines() do
|
||||||
local status, file = line:match('(..) (.*)')
|
local status, file = line:match('(..) (.*)')
|
||||||
|
assert(status:sub(2, 2) == ' ')
|
||||||
status = trim(status)
|
status = trim(status)
|
||||||
if file:sub(1, 1) == '"' then
|
if file:sub(1, 1) == '"' then
|
||||||
-- filename with a space is quoted by git
|
-- filename with a space is quoted by git
|
||||||
|
@ -445,7 +466,23 @@ local function gitStatus()
|
||||||
end
|
end
|
||||||
file = 'usr/' .. file
|
file = 'usr/' .. file
|
||||||
if not fileExists(file) then
|
if not fileExists(file) then
|
||||||
log('Missing file: %q', file)
|
if status == 'D' then
|
||||||
|
local prev_owner = assert(file2item[file])
|
||||||
|
if prev_owner == item then
|
||||||
|
log('Item %s removed %q installed by itself',
|
||||||
|
item, file)
|
||||||
|
elseif isDependency(prev_owner, item, item2deps) then
|
||||||
|
log('Item %s removed %q installed by its follower %s',
|
||||||
|
item, file, prev_owner)
|
||||||
|
else
|
||||||
|
log('Item %s removed %q installed by %s',
|
||||||
|
item, file, prev_owner)
|
||||||
|
end
|
||||||
|
elseif isSymlink(file) then
|
||||||
|
log('Broken symlink: %q', file)
|
||||||
|
else
|
||||||
|
log('Missing file: %q', file)
|
||||||
|
end
|
||||||
elseif not isBlacklisted(file) then
|
elseif not isBlacklisted(file) then
|
||||||
if status == 'A' then
|
if status == 'A' then
|
||||||
table.insert(new_files, file)
|
table.insert(new_files, file)
|
||||||
|
@ -531,13 +568,16 @@ end
|
||||||
|
|
||||||
local function removeEmptyDirs(item)
|
local function removeEmptyDirs(item)
|
||||||
-- removing an empty dir can reveal another one (parent)
|
-- removing an empty dir can reveal another one (parent)
|
||||||
|
-- don't pass item to mute the log message
|
||||||
local go_on = true
|
local go_on = true
|
||||||
while go_on do
|
while go_on do
|
||||||
go_on = false
|
go_on = false
|
||||||
local f = io.popen('find usr/* -empty -type d', 'r')
|
local f = io.popen('find usr/* -empty -type d', 'r')
|
||||||
for dir in f:lines() do
|
for dir in f:lines() do
|
||||||
log("Remove empty directory %s created by %s",
|
if item then
|
||||||
dir, item)
|
log("Remove empty directory %s created by %s",
|
||||||
|
dir, item)
|
||||||
|
end
|
||||||
os.remove(dir)
|
os.remove(dir)
|
||||||
go_on = true
|
go_on = true
|
||||||
end
|
end
|
||||||
|
@ -545,20 +585,90 @@ local function removeEmptyDirs(item)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
local function prepareTree(pass, item, item2deps, prev_files, item2index)
|
||||||
|
if pass == 'first' then
|
||||||
|
gitCheckout(
|
||||||
|
itemToBranch(item, pass),
|
||||||
|
item2deps[item],
|
||||||
|
item2index,
|
||||||
|
pass
|
||||||
|
)
|
||||||
|
elseif pass == 'second' then
|
||||||
|
-- Build item second time to check if it builds correctly if
|
||||||
|
-- its followers and unrelated packages have been built.
|
||||||
|
gitCheckout(
|
||||||
|
itemToBranch(item, 'second'),
|
||||||
|
{GIT_ALL_PSEUDOITEM},
|
||||||
|
item2index,
|
||||||
|
'first'
|
||||||
|
)
|
||||||
|
removeEmptyDirs()
|
||||||
|
if prev_files then
|
||||||
|
-- Remove files of item from previous build.
|
||||||
|
for _, file in ipairs(prev_files) do
|
||||||
|
os.remove(file)
|
||||||
|
end
|
||||||
|
gitAdd()
|
||||||
|
gitCommit(("Remove %s to rebuild it"):format(item, pass))
|
||||||
|
end
|
||||||
|
else
|
||||||
|
error("Unknown pass: " .. pass)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
local function comparePasses(item, new_files, prev_file2item, prev_files)
|
||||||
|
local files_set = {}
|
||||||
|
for _, file in ipairs(new_files) do
|
||||||
|
if not prev_file2item[file] then
|
||||||
|
log('Item %s installs a file on second pass only: %s',
|
||||||
|
item, file)
|
||||||
|
elseif prev_file2item[file] ~= item then
|
||||||
|
log('File %s was installed by %s on first pass ' ..
|
||||||
|
'and by %s - on the second pass',
|
||||||
|
file, prev_file2item[file], item)
|
||||||
|
end
|
||||||
|
files_set[file] = true
|
||||||
|
end
|
||||||
|
for _, file in ipairs(prev_files) do
|
||||||
|
if not files_set[file] then
|
||||||
|
log('Item %s installs a file on first pass only: %s',
|
||||||
|
item, file)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
-- TODO compare contents of files (nm for binaries)
|
||||||
|
end
|
||||||
|
|
||||||
|
local function isBuilt(item, files)
|
||||||
|
local target, pkg = parseItem(item)
|
||||||
|
local INSTALLED = 'usr/%s/installed/%s'
|
||||||
|
local installed = INSTALLED:format(target, pkg)
|
||||||
|
for _, file in ipairs(files) do
|
||||||
|
if file == installed then
|
||||||
|
return true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return false
|
||||||
|
end
|
||||||
|
|
||||||
-- builds package, returns list of new files
|
-- builds package, returns list of new files
|
||||||
local function buildItem(item, item2deps, file2item, item2index)
|
-- prev_files is passed only to second pass.
|
||||||
gitCheckout(itemToBranch(item), item2deps[item], item2index)
|
local function buildItem(item, item2deps, file2item, item2index, pass, prev_files)
|
||||||
|
prepareTree(pass, item, item2deps, prev_files, item2index)
|
||||||
local target, pkg = parseItem(item)
|
local target, pkg = parseItem(item)
|
||||||
local cmd = '%s %s MXE_TARGETS=%s --jobs=1'
|
local cmd = '%s %s MXE_TARGETS=%s --jobs=1'
|
||||||
os.execute(cmd:format(tool 'make', pkg, target))
|
os.execute(cmd:format(tool 'make', pkg, target))
|
||||||
gitAdd()
|
gitAdd()
|
||||||
local new_files, changed_files = gitStatus()
|
local new_files, changed_files = gitStatus(item, item2deps, file2item)
|
||||||
if #new_files + #changed_files > 0 then
|
if #new_files + #changed_files > 0 then
|
||||||
gitCommit(("Build %s"):format(item))
|
gitCommit(("Build %s, pass %s"):format(item, pass))
|
||||||
end
|
end
|
||||||
for _, file in ipairs(new_files) do
|
if pass == 'first' then
|
||||||
checkFile(file, item)
|
for _, file in ipairs(new_files) do
|
||||||
file2item[file] = item
|
checkFile(file, item)
|
||||||
|
file2item[file] = item
|
||||||
|
end
|
||||||
|
elseif isBuilt(item, new_files) then
|
||||||
|
comparePasses(item, new_files, file2item, prev_files)
|
||||||
end
|
end
|
||||||
for _, file in ipairs(changed_files) do
|
for _, file in ipairs(changed_files) do
|
||||||
checkFile(file, item)
|
checkFile(file, item)
|
||||||
|
@ -567,8 +677,8 @@ local function buildItem(item, item2deps, file2item, item2index)
|
||||||
if not isInArray(creator_item, item2deps[item]) then
|
if not isInArray(creator_item, item2deps[item]) then
|
||||||
table.insert(item2deps[item], creator_item)
|
table.insert(item2deps[item], creator_item)
|
||||||
end
|
end
|
||||||
log('Item %s changes %s, created by %s',
|
log('Item %s (pass %s) changes %s, created by %s',
|
||||||
item, file, creator_item)
|
item, pass, file, creator_item)
|
||||||
end
|
end
|
||||||
checkFileList(concatArrays(new_files, changed_files), item)
|
checkFileList(concatArrays(new_files, changed_files), item)
|
||||||
removeEmptyDirs(item)
|
removeEmptyDirs(item)
|
||||||
|
@ -687,18 +797,6 @@ local function makeDeb(item, files, deps, ver)
|
||||||
makePackage(deb_pkg, files, deb_deps, ver, d1, d2)
|
makePackage(deb_pkg, files, deb_deps, ver, d1, d2)
|
||||||
end
|
end
|
||||||
|
|
||||||
local function isBuilt(item, files)
|
|
||||||
local target, pkg = parseItem(item)
|
|
||||||
local INSTALLED = 'usr/%s/installed/%s'
|
|
||||||
local installed = INSTALLED:format(target, pkg)
|
|
||||||
for _, file in ipairs(files) do
|
|
||||||
if file == installed then
|
|
||||||
return true
|
|
||||||
end
|
|
||||||
end
|
|
||||||
return false
|
|
||||||
end
|
|
||||||
|
|
||||||
local function findForeignInstalls(item, files)
|
local function findForeignInstalls(item, files)
|
||||||
for _, file in ipairs(files) do
|
for _, file in ipairs(files) do
|
||||||
local pattern = 'usr/([^/]+)/installed/([^/]+)'
|
local pattern = 'usr/([^/]+)/installed/([^/]+)'
|
||||||
|
@ -771,7 +869,8 @@ local function isEmpty(files)
|
||||||
end
|
end
|
||||||
|
|
||||||
-- build all packages, save filelist to list file
|
-- build all packages, save filelist to list file
|
||||||
local function buildPackages(items, item2deps)
|
-- prev_files is passed only to second pass.
|
||||||
|
local function buildPackages(items, item2deps, pass, prev_item2files)
|
||||||
local broken = {}
|
local broken = {}
|
||||||
local unbroken = {}
|
local unbroken = {}
|
||||||
local file2item = {}
|
local file2item = {}
|
||||||
|
@ -784,12 +883,23 @@ local function buildPackages(items, item2deps)
|
||||||
end
|
end
|
||||||
return false
|
return false
|
||||||
end
|
end
|
||||||
|
if pass == 'second' then
|
||||||
|
assert(prev_item2files)
|
||||||
|
-- fill file2item with data from prev_item2files
|
||||||
|
for item, files in pairs(prev_item2files) do
|
||||||
|
for _, file in ipairs(files) do
|
||||||
|
file2item[file] = item
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
local item2index = makeItem2Index(items)
|
local item2index = makeItem2Index(items)
|
||||||
local progress_printer = progressPrinter(items)
|
local progress_printer = progressPrinter(items)
|
||||||
for i, item in ipairs(items) do
|
for i, item in ipairs(items) do
|
||||||
if not brokenDep(item) then
|
if not brokenDep(item) then
|
||||||
local files = buildItem(item, item2deps,
|
local prev_files = prev_item2files and prev_item2files[item]
|
||||||
file2item, item2index)
|
local files = buildItem(
|
||||||
|
item, item2deps, file2item, item2index, pass, prev_files
|
||||||
|
)
|
||||||
findForeignInstalls(item, files)
|
findForeignInstalls(item, files)
|
||||||
if isBuilt(item, files) then
|
if isBuilt(item, files) then
|
||||||
item2files[item] = files
|
item2files[item] = files
|
||||||
|
@ -917,36 +1027,59 @@ local function makeMxeSourcePackage()
|
||||||
makePackage(name, files, deps, ver, d1, d2)
|
makePackage(name, files, deps, ver, d1, d2)
|
||||||
end
|
end
|
||||||
|
|
||||||
assert(not io.open('usr/.git'), 'Remove usr/')
|
local function main()
|
||||||
local MXE_DIR_EXPECTED = '/usr/lib/mxe'
|
assert(not io.open('usr/.git'), 'Remove usr/')
|
||||||
if MXE_DIR ~= MXE_DIR_EXPECTED then
|
local MXE_DIR_EXPECTED = '/usr/lib/mxe'
|
||||||
log("Warning! Building in dir %s, not in %s",
|
if MXE_DIR ~= MXE_DIR_EXPECTED then
|
||||||
MXE_DIR, MXE_DIR_EXPECTED)
|
log("Warning! Building in dir %s, not in %s",
|
||||||
end
|
MXE_DIR, MXE_DIR_EXPECTED)
|
||||||
gitInit()
|
end
|
||||||
assert(execute(("%s check-requirements MXE_TARGETS=%q"):format(
|
gitInit()
|
||||||
tool 'make', table.concat(TARGETS, ' '))))
|
assert(execute(("%s check-requirements MXE_TARGETS=%q"):format(
|
||||||
if not max_items then
|
tool 'make', table.concat(TARGETS, ' '))))
|
||||||
local cmd = ('%s download -j 6 -k'):format(tool 'make')
|
if not max_items then
|
||||||
while not execute(cmd) do end
|
local cmd = ('%s download -j 6 -k'):format(tool 'make')
|
||||||
end
|
while not execute(cmd) do end
|
||||||
gitAdd()
|
end
|
||||||
gitCommit('Initial commit')
|
gitAdd()
|
||||||
gitTag(GIT_INITIAL)
|
gitCommit('Initial commit')
|
||||||
local items, item2deps, item2ver = getItems()
|
gitTag(GIT_INITIAL)
|
||||||
local build_list = sortForBuild(items, item2deps)
|
local items, item2deps, item2ver = getItems()
|
||||||
assert(isTopoOrdered(build_list, items, item2deps))
|
local build_list = sortForBuild(items, item2deps)
|
||||||
build_list = sliceArray(build_list, max_items)
|
assert(isTopoOrdered(build_list, items, item2deps))
|
||||||
local unbroken, item2files = buildPackages(build_list, item2deps)
|
build_list = sliceArray(build_list, max_items)
|
||||||
gitCheckout(GIT_ALL, unbroken, makeItem2Index(build_list))
|
local first_pass_failed, second_pass_failed
|
||||||
makeDebs(unbroken, item2deps, item2ver, item2files)
|
local unbroken, item2files = buildPackages(
|
||||||
if not no_debs then
|
build_list, item2deps, 'first'
|
||||||
makeMxeRequirementsPackage('wheezy')
|
)
|
||||||
makeMxeRequirementsPackage('jessie')
|
if #unbroken < #build_list then
|
||||||
end
|
first_pass_failed = true
|
||||||
makeMxeSourcePackage()
|
end
|
||||||
if #unbroken < #build_list then
|
gitCheckout(
|
||||||
local code = 1
|
itemToBranch(GIT_ALL_PSEUDOITEM, 'first'),
|
||||||
local close = true
|
unbroken,
|
||||||
os.exit(code, close)
|
makeItem2Index(build_list),
|
||||||
|
'first'
|
||||||
|
)
|
||||||
|
makeDebs(unbroken, item2deps, item2ver, item2files)
|
||||||
|
if not no_debs then
|
||||||
|
makeMxeRequirementsPackage('wheezy')
|
||||||
|
makeMxeRequirementsPackage('jessie')
|
||||||
|
end
|
||||||
|
makeMxeSourcePackage()
|
||||||
|
if not no_second_pass then
|
||||||
|
local unbroken_second = buildPackages(
|
||||||
|
build_list, item2deps, 'second', item2files
|
||||||
|
)
|
||||||
|
if #unbroken_second < #build_list then
|
||||||
|
second_pass_failed = true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
if first_pass_failed or second_pass_failed then
|
||||||
|
local code = 1
|
||||||
|
local close = true
|
||||||
|
os.exit(code, close)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
main()
|
||||||
|
|
Loading…
Reference in New Issue