You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

507 lines
16 KiB

#!/usr/libexec/flua
--[[
SPDX-License-Identifier: BSD-2-Clause-FreeBSD
Copyright (c) 2022 Stefan Esser <se@FreeBSD.org>
Generate a list of existing and required CONFLICTS_INSTALL lines
for all ports (limited to ports for which official packages are
provided).
This script depends on the ports-mgmt/pkg-provides port for the list
of files installed by all pre-built packages for the architecture
the script is run on.
The script generates a list of ports by running "pkg provides ." and
a mapping from package base name to origin via "pkg rquery '%n %o'".
The existing CONFLICTS and CONFLICTS_INSTALL definitions are fetched
by "make -C $origin -V CONFLICTS -V CONFLICTS_INSTALL". This list is
only representative for the options configured for each port (i.e.
if non-default options have been selected and registered, these may
lead to a non-default list of conflicts).
The script detects files used by more than one port, than lists by
origin the existing definition and the list of package base names
that have been detected to cause install conflicts followed by the
list of duplicate files separated by a hash character "#".
This script uses the "hidden" LUA interpreter in the FreeBSD base
systems and does not need any port except "pkg-provides" to be run.
The run-time on my system checking the ~32000 packages available
for -CURRENT on amd64 is less than 250 seconds.
Example output:
# Port: games/sol
# Files: bin/sol
# < aisleriot gnome-games
# > aisleriot
portedit merge -ie 'CONFLICTS_INSTALL=aisleriot # bin/sol' /usr/ports/games/sol
The output is per port (for all flavors of the port, if applicable),
gives examples of conflicting files (mostly to understand whether
different versions of a port could co-exist), the current CONFLICTS
and CONFLICTS_INSTALL entries merged, and a suggested new entry.
This information is followed by a portedit command line that should
do the right thing for simple cases, but the result should always
be checked before the resulting Makefile is committed.
--]]
require "lfs"
-------------------------------------------------------------------
local file_pattern = "."
local database = "/var/db/pkg/provides/provides.db"
local max_age = 1 * 24 * 3600 -- maximum age of database file in seconds
-------------------------------------------------------------------
local function table_sorted_keys(t)
local result = {}
for k, _ in pairs(t) do
result[#result + 1] = k
end
table.sort(result)
return result
end
-------------------------------------------------------------------
local function table_sort_uniq(t)
local result = {}
if t then
local last
table.sort(t)
for _, entry in ipairs(t) do
if entry ~= last then
last = entry
result[#result + 1] = entry
end
end
end
return result
end
-------------------------------------------------------------------
local function fnmatch(name, pattern)
local function fnsubst(s)
s = string.gsub(s, "%%", "%%%%")
s = string.gsub(s, "%+", "%%+")
s = string.gsub(s, "%-", "%%-")
s = string.gsub(s, "%.", "%%.")
s = string.gsub(s, "%?", ".")
s = string.gsub(s, "%*", ".*")
return s
end
local rexpr = ""
local left, middle, right
while true do
left, middle, right = string.match(pattern, "([^[]*)(%[[^]]+%])(.*)")
if not left then
break
end
rexpr = rexpr .. fnsubst(left) .. middle
pattern = right
end
rexpr = "^" .. rexpr .. fnsubst(pattern) .. "$"
return string.find(name, rexpr)
end
-------------------------------------------------------------------
local function fetch_pkgs_origins()
local pkgs = {}
local pipe = io.popen("pkg rquery '%n %o'")
for line in pipe:lines() do
local pkgbase, origin = string.match(line, "(%S+) (%S+)")
pkgs[origin] = pkgbase
end
pipe:close()
pipe = io.popen("pkg rquery '%n %o %At %Av'")
for line in pipe:lines() do
local pkgbase, origin, tag, value = string.match(line, "(%S+) (%S+) (%S+) (%S+)")
if tag == "flavor" then
pkgs[origin] = nil
pkgs[origin .. "@" .. value] = pkgbase
end
end
pipe:close()
return pkgs
end
-------------------------------------------------------------------
local BAD_FILE_PATTERN = {
"^[^/]+$",
"^lib/python[%d%.]+/site%-packages/examples/[^/]+$",
"^lib/python[%d%.]+/site%-packages/samples/[^/]+$",
"^lib/python[%d%.]+/site%-packages/test/[^/]+$",
"^lib/python[%d%.]+/site%-packages/test_app/[^/]+$",
"^lib/python[%d%.]+/site%-packages/tests/[^/]+$",
"^lib/python[%d%.]+/site%-packages/tests/unit/[^/]+$",
}
local BAD_FILE_PKGS = {}
local function check_bad_file(pkgbase, file)
for _, pattern in ipairs(BAD_FILE_PATTERN) do
if string.match(file, pattern) then
BAD_FILE_PKGS[pkgbase] = BAD_FILE_PKGS[pkgbase] or {}
table.insert(BAD_FILE_PKGS[pkgbase], file)
break
end
end
end
-------------------------------------------------------------------
local function read_files(pattern)
local files_table = {}
local now = os.time()
local modification_time = lfs.attributes(database, "modification")
if not modification_time then
print("# Aborting: package file database " .. database .. " does not exist.")
print("# Install the 'pkg-provides' package and add it as a module to 'pkg.conf'.")
print("# Then fetch the database with 'pkg update' or 'pkg provides -u'.")
os.exit(1)
end
if now - modification_time > max_age then
print("# Aborting: package file database " .. database)
print("# is older than " .. max_age .. " seconds.")
print("# Use 'pkg provides -u' to update the database.")
os.exit(2)
end
local pipe = io.popen("locate -d " .. database .. " " .. pattern)
if pipe then
for line in pipe:lines() do
local pkgbase, file = string.match(line, "([^*]+)%*([^*]+)")
if file:sub(1, 11) == "/usr/local/" then
file = file:sub(12)
end
check_bad_file(pkgbase, file)
local t = files_table[file] or {}
t[#t + 1] = pkgbase
files_table[file] = t
end
pipe:close()
end
return files_table
end
-------------------------------------------------------------------
local DUPLICATE_FILE = {}
local function fetch_pkg_pairs(pattern)
local pkg_pairs = {}
for file, pkgbases in pairs(read_files(pattern)) do
if #pkgbases >= 2 then
DUPLICATE_FILE[file] = true
for i = 1, #pkgbases -1 do
local pkg_i = pkgbases[i]
for j = i + 1, #pkgbases do
local pkg_j = pkgbases[j]
if pkg_i ~= pkg_j then
local p1 = pkg_pairs[pkg_i] or {}
local p2 = p1[pkg_j] or {}
p2[#p2 + 1] = file
p1[pkg_j] = p2
pkg_pairs[pkg_i] = p1
end
end
end
end
end
return pkg_pairs
end
-------------------------------------------------------------------
local function conflicts_delta(old, new)
local old_seen = {}
local changed
for i = 1, #new do
local matched
for j = 1, #old do
if new[i] == old[j] or fnmatch(new[i], old[j]) then
new[i] = old[j]
old_seen[j] = true
matched = true
break
end
end
changed = changed or not matched
end
if not changed then
for j = 1, #old do
if not old_seen[j] then
changed = true
break
end
end
end
if changed then
return table_sort_uniq(new)
end
end
-------------------------------------------------------------------
local function fetch_port_conflicts(origin)
local dir, flavor = origin:match("([^@]+)@?(.*)")
if flavor ~= "" then
flavor = " FLAVOR=" .. flavor
end
local seen = {}
local portdir = "/usr/ports/" .. dir
local pipe = io.popen("make -C " .. portdir .. flavor .. " -V CONFLICTS -V CONFLICTS_INSTALL 2>/dev/null")
for line in pipe:lines() do
for word in line:gmatch("(%S+)%s?") do
seen[word] = true
end
end
pipe:close()
return table_sorted_keys(seen)
end
-------------------------------------------------------------------
local function conflicting_pkgs(conflicting)
local pkgs = {}
for origin, pkgbase in pairs(fetch_pkgs_origins()) do
if conflicting[pkgbase] then
pkgs[origin] = pkgbase
end
end
return pkgs
end
-------------------------------------------------------------------
local function collect_conflicts(pkg_pairs)
local pkgs = {}
for pkg_i, p1 in pairs(pkg_pairs) do
for pkg_j, _ in pairs(p1) do
pkgs[pkg_i] = pkgs[pkg_i] or {}
table.insert(pkgs[pkg_i], pkg_j)
pkgs[pkg_j] = pkgs[pkg_j] or {}
table.insert(pkgs[pkg_j], pkg_i)
end
end
return pkgs
end
-------------------------------------------------------------------
local function split_origins(origin_list)
local port_list = {}
local flavors = {}
local last_port
for _, origin in ipairs(origin_list) do
local port, flavor = string.match(origin, "([^@]+)@?(.*)")
if port ~= last_port then
port_list[#port_list + 1] = port
if flavor ~= "" then
flavors[port] = {flavor}
end
else
table.insert(flavors[port], flavor)
end
last_port = port
end
return port_list, flavors
end
-------------------------------------------------------------------
local PKG_PAIR_FILES = fetch_pkg_pairs(file_pattern)
local CONFLICT_PKGS = collect_conflicts(PKG_PAIR_FILES)
local PKGBASE = conflicting_pkgs(CONFLICT_PKGS)
local ORIGIN_LIST = table_sorted_keys(PKGBASE)
local PORT_LIST, FLAVORS = split_origins(ORIGIN_LIST)
local function conflicting_files(pkg_i, pkgs)
local files = {}
local all_files = {}
local f
local p1 = PKG_PAIR_FILES[pkg_i]
if p1 then
for _, pkg_j in ipairs(pkgs) do
f = p1[pkg_j]
if f then
table.sort(f)
files[#files + 1] = f[1]
table.move(f, 1, #f, #all_files + 1, all_files)
end
end
end
for _, pkg_j in ipairs(pkgs) do
p1 = PKG_PAIR_FILES[pkg_j]
f = p1 and p1[pkg_i]
if f then
table.sort(f)
files[#files + 1] = f[1]
table.move(f, 1, #f, #all_files + 1, all_files)
end
end
return table_sort_uniq(files), table_sort_uniq(all_files)
end
---------------------------------------------------------------------
local version_pattern = {
"^lib/python%d%.%d/",
"^share/py3%d%d%?-",
"^share/%a+/py3%d%d%?-",
"^lib/lua/%d%.%d/",
"^share/lua/%d%.%d/",
"^lib/perl5/[%d%.]+/",
"^lib/perl5/site_perl/mach/[%d%.]+/",
"^lib/ruby/gems/%d%.%d/",
"^lib/ruby/site_ruby/%d%.%d/",
}
local function generalize_patterns(pkgs, files)
local function match_any(str, pattern_array)
for _, pattern in ipairs(pattern_array) do
if string.match(str, pattern) then
return true
end
end
return false
end
local function unversioned_files()
for i = 1, #files do
if not match_any(files[i], version_pattern) then
return true
end
end
return false
end
local function pkg_wildcards(from, ...)
local to_list = {...}
local result = {}
for i = 1, #pkgs do
local orig_pkg = pkgs[i]
for _, to in ipairs(to_list) do
result[string.gsub(orig_pkg, from, to)] = true
end
end
pkgs = table_sorted_keys(result)
end
local pkg_pfx_php = "php[0-9][0-9]-"
local pkg_sfx_php = "-php[0-9][0-9]"
local pkg_pfx_python2
local pkg_sfx_python2
local pkg_pfx_python3
local pkg_sfx_python3
local pkg_pfx_lua
local pkg_pfx_ruby
pkgs = table_sort_uniq(pkgs)
if unversioned_files() then
pkg_pfx_python2 = "py3[0-9]-" -- e.g. py39-
pkg_sfx_python2 = "-py3[0-9]"
pkg_pfx_python3 = "py3[0-9][0-9]-" -- e.g. py311-
pkg_sfx_python3 = "-py3[0-9][0-9]"
pkg_pfx_lua = "lua[0-9][0-9]-"
pkg_pfx_ruby = "ruby[0-9][0-9]-"
else
pkg_pfx_python2 = "${PYTHON_PKGNAMEPREFIX}"
pkg_sfx_python2 = "${PYTHON_PKGNAMESUFFIX}"
pkg_pfx_python3 = nil
pkg_sfx_python3 = nil
pkg_pfx_lua = "${LUA_PKGNAMEPREFIX}"
pkg_pfx_ruby = "${RUBY_PKGNAMEPREFIX}"
end
pkg_wildcards("^php%d%d%-", pkg_pfx_php)
pkg_wildcards("-php%d%d$", pkg_sfx_php)
pkg_wildcards("^phpunit%d%-", "phpunit[0-9]-")
pkg_wildcards("^py3%d%-", pkg_pfx_python2, pkg_pfx_python3)
pkg_wildcards("-py3%d%$", pkg_sfx_python2, pkg_sfx_python3)
pkg_wildcards("^py3%d%d%-", pkg_pfx_python2, pkg_pfx_python3)
pkg_wildcards("-py3%d%d%$", pkg_sfx_python2, pkg_sfx_python3)
pkg_wildcards("^lua%d%d%-", pkg_pfx_lua)
pkg_wildcards("-emacs_[%a_]*", "-emacs_*")
pkg_wildcards("^ghostscript%d%-", "ghostscript[0-9]-")
pkg_wildcards("^bacula%d%-", "bacula[0-9]-")
pkg_wildcards("^bacula%d%d%-", "bacula[0-9][0-9]-")
pkg_wildcards("^bareos%d%-", "bareos[0-9]-")
pkg_wildcards("^bareos%d%d%-", "bareos[0-9][0-9]-")
pkg_wildcards("^moosefs%d%-", "moosefs[0-9]-")
pkg_wildcards("^ruby%d+-", pkg_pfx_ruby)
return table_sort_uniq(pkgs)
end
-------------------------------------------------------------------
for _, port in ipairs(PORT_LIST) do
local function merge_table(t1, t2)
table.move(t2, 1, #t2, #t1 + 1, t1)
end
local port_conflicts = {}
local files = {}
local msg_files = {}
local conflict_pkgs = {}
local function merge_data(origin)
local pkgbase = PKGBASE[origin]
if not BAD_FILE_PKGS[pkgbase] then
local pkg_confl_file1, pkg_confl_files = conflicting_files(pkgbase, CONFLICT_PKGS[pkgbase])
merge_table(msg_files, pkg_confl_file1) -- 1 file per flavor
merge_table(files, pkg_confl_files) -- all conflicting files
merge_table(conflict_pkgs, CONFLICT_PKGS[pkgbase])
merge_table(port_conflicts, fetch_port_conflicts(origin))
end
end
local flavors = FLAVORS[port]
if flavors then
for _, flavor in ipairs(flavors) do
merge_data(port .. "@" .. flavor)
end
else
merge_data(port)
end
files = table_sort_uniq(files)
msg_files = table_sort_uniq(msg_files)
conflict_pkgs = generalize_patterns(conflict_pkgs, files)
if #port_conflicts then
port_conflicts = table_sort_uniq(port_conflicts)
conflict_pkgs = conflicts_delta(port_conflicts, conflict_pkgs)
end
if conflict_pkgs then
local conflicts_string_cur = table.concat(port_conflicts, " ")
local conflicts_string_new = table.concat(conflict_pkgs, " ")
local file_list = table.concat(msg_files, " ")
print("# Port: " .. port)
print("# Files: " .. file_list)
if conflicts_string_cur ~= "" then
print("# < " .. conflicts_string_cur)
end
print("# > " .. conflicts_string_new)
print("portedit merge -ie 'CONFLICTS_INSTALL=" .. conflicts_string_new ..
" # " .. file_list .. "' /usr/ports/" .. port)
print()
end
end
-------------------------------------------------------------------
local BAD_FILES_ORIGINS = {}
for _, origin in ipairs(ORIGIN_LIST) do
local pkgbase = PKGBASE[origin]
local files = BAD_FILE_PKGS[pkgbase]
if files then
for _, file in ipairs(files) do
if DUPLICATE_FILE[file] then
local port = string.match(origin, "([^@]+)@?")
BAD_FILES_ORIGINS[port] = BAD_FILES_ORIGINS[origin] or {}
table.insert(BAD_FILES_ORIGINS[port], file)
end
end
end
end
-------------------------------------------------------------------
local bad_origins = table_sorted_keys(BAD_FILES_ORIGINS)
if #bad_origins > 0 then
print ("# Ports with badly named files:")
print ()
for _, port in ipairs(bad_origins) do
print ("# " .. port)
local files = BAD_FILES_ORIGINS[port]
table.sort(files)
for _, file in ipairs(files) do
print ("#", file)
end
print()
end
end